From f8c30a678a6803bc183ba4bd969d50c889599595 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Fri, 9 Mar 2018 19:32:45 -0800 Subject: [PATCH 01/82] Reorganize packages, closes #79 - All packages go into either cmd or pkg directories - Most packages renamed - Server packages moved into pkg/server - sanitize moved into webui, as that's the only place it's used - filestore moved into pkg/storage/file - Makefile updated, and PKG variable use fixed --- .gitignore | 4 ++ Makefile | 31 +++++----- cmd/client/list.go | 2 +- cmd/client/match.go | 2 +- cmd/client/mbox.go | 2 +- inbucket.go => cmd/inbucket/main.go | 34 +++++------ {config => pkg/config}/config.go | 0 {log => pkg/log}/logging.go | 0 {log => pkg/log}/metrics.go | 0 {log => pkg/log}/stdout_unix.go | 0 {log => pkg/log}/stdout_windows.go | 0 {msghub => pkg/msghub}/hub.go | 0 {msghub => pkg/msghub}/hub_test.go | 0 {rest => pkg/rest}/apiv1_controller.go | 28 +++++----- {rest => pkg/rest}/apiv1_controller_test.go | 2 +- {rest => pkg/rest}/client/apiv1_client.go | 2 +- .../rest}/client/apiv1_client_test.go | 0 {rest => pkg/rest}/client/rest.go | 0 {rest => pkg/rest}/client/rest_test.go | 0 {rest => pkg/rest}/model/apiv1_model.go | 0 pkg/rest/routes.go | 23 ++++++++ {rest => pkg/rest}/socketv1_controller.go | 22 ++++---- {rest => pkg/rest}/testutils_test.go | 14 ++--- {pop3d => pkg/server/pop3}/handler.go | 6 +- {pop3d => pkg/server/pop3}/listener.go | 8 +-- {smtpd => pkg/server/smtp}/handler.go | 10 ++-- {smtpd => pkg/server/smtp}/handler_test.go | 8 +-- {smtpd => pkg/server/smtp}/listener.go | 10 ++-- {httpd => pkg/server/web}/context.go | 8 +-- {httpd => pkg/server/web}/helpers.go | 4 +- {httpd => pkg/server/web}/helpers_test.go | 2 +- {httpd => pkg/server/web}/rest.go | 2 +- {httpd => pkg/server/web}/server.go | 12 ++-- {httpd => pkg/server/web}/template.go | 4 +- {datastore => pkg/storage}/datastore.go | 0 {filestore => pkg/storage/file}/fmessage.go | 4 +- {filestore => pkg/storage/file}/fstore.go | 8 +-- .../storage/file}/fstore_test.go | 2 +- {datastore => pkg/storage}/lock.go | 0 {datastore => pkg/storage}/lock_test.go | 2 +- {datastore => pkg/storage}/retention.go | 4 +- {datastore => pkg/storage}/retention_test.go | 0 {datastore => pkg/storage}/testing.go | 0 {stringutil => pkg/stringutil}/utils.go | 0 {stringutil => pkg/stringutil}/utils_test.go | 0 {webui => pkg/webui}/mailbox_controller.go | 56 +++++++++---------- {webui => pkg/webui}/recent.go | 6 +- {webui => pkg/webui}/root_controller.go | 28 +++++----- pkg/webui/routes.go | 35 ++++++++++++ {sanitize => pkg/webui/sanitize}/css.go | 0 {sanitize => pkg/webui/sanitize}/css_test.go | 0 {sanitize => pkg/webui/sanitize}/html.go | 0 {sanitize => pkg/webui/sanitize}/html_test.go | 2 +- rest/routes.go | 23 -------- webui/routes.go | 35 ------------ 55 files changed, 225 insertions(+), 220 deletions(-) rename inbucket.go => cmd/inbucket/main.go (83%) rename {config => pkg/config}/config.go (100%) rename {log => pkg/log}/logging.go (100%) rename {log => pkg/log}/metrics.go (100%) rename {log => pkg/log}/stdout_unix.go (100%) rename {log => pkg/log}/stdout_windows.go (100%) rename {msghub => pkg/msghub}/hub.go (100%) rename {msghub => pkg/msghub}/hub_test.go (100%) rename {rest => pkg/rest}/apiv1_controller.go (91%) rename {rest => pkg/rest}/apiv1_controller_test.go (99%) rename {rest => pkg/rest}/client/apiv1_client.go (98%) rename {rest => pkg/rest}/client/apiv1_client_test.go (100%) rename {rest => pkg/rest}/client/rest.go (100%) rename {rest => pkg/rest}/client/rest_test.go (100%) rename {rest => pkg/rest}/model/apiv1_model.go (100%) create mode 100644 pkg/rest/routes.go rename {rest => pkg/rest}/socketv1_controller.go (88%) rename {rest => pkg/rest}/testutils_test.go (95%) rename {pop3d => pkg/server/pop3}/handler.go (99%) rename {pop3d => pkg/server/pop3}/listener.go (95%) rename {smtpd => pkg/server/smtp}/handler.go (98%) rename {smtpd => pkg/server/smtp}/handler_test.go (98%) rename {smtpd => pkg/server/smtp}/listener.go (96%) rename {httpd => pkg/server/web}/context.go (90%) rename {httpd => pkg/server/web}/helpers.go (97%) rename {httpd => pkg/server/web}/helpers_test.go (98%) rename {httpd => pkg/server/web}/rest.go (96%) rename {httpd => pkg/server/web}/server.go (93%) rename {httpd => pkg/server/web}/template.go (97%) rename {datastore => pkg/storage}/datastore.go (100%) rename {filestore => pkg/storage/file}/fmessage.go (98%) rename {filestore => pkg/storage/file}/fstore.go (98%) rename {filestore => pkg/storage/file}/fstore_test.go (99%) rename {datastore => pkg/storage}/lock.go (100%) rename {datastore => pkg/storage}/lock_test.go (96%) rename {datastore => pkg/storage}/retention.go (98%) rename {datastore => pkg/storage}/retention_test.go (100%) rename {datastore => pkg/storage}/testing.go (100%) rename {stringutil => pkg/stringutil}/utils.go (100%) rename {stringutil => pkg/stringutil}/utils_test.go (100%) rename {webui => pkg/webui}/mailbox_controller.go (84%) rename {webui => pkg/webui}/recent.go (83%) rename {webui => pkg/webui}/root_controller.go (75%) create mode 100644 pkg/webui/routes.go rename {sanitize => pkg/webui/sanitize}/css.go (100%) rename {sanitize => pkg/webui/sanitize}/css_test.go (100%) rename {sanitize => pkg/webui/sanitize}/html.go (100%) rename {sanitize => pkg/webui/sanitize}/html_test.go (98%) delete mode 100644 rest/routes.go delete mode 100644 webui/routes.go diff --git a/.gitignore b/.gitignore index 52aeb50..dc71557 100644 --- a/.gitignore +++ b/.gitignore @@ -26,8 +26,12 @@ _testmain.go *.swo # our binaries +/client +/client.exe /inbucket /inbucket.exe /dist/** /cmd/client/client /cmd/client/client.exe +/cmd/inbucket/inbucket +/cmd/inbucket/inbucket.exe diff --git a/Makefile b/Makefile index 8101866..ae01ee8 100644 --- a/Makefile +++ b/Makefile @@ -1,26 +1,27 @@ -PKG := inbucket -SHELL := /bin/sh +SHELL = /bin/sh -SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*") -PKGS := $$(go list ./... | grep -v /vendor/) +SRC ::= $(shell find . -type f -name '*.go' -not -path "./vendor/*") +PKGS ::= $(shell go list ./... | grep -v /vendor/) -.PHONY: all build clean fmt install lint simplify test +.PHONY: all build clean fmt lint simplify test -all: test lint build +commands ::= client inbucket + +all: clean test lint build + +$(commands): %: cmd/% + go build ./$< clean: - go clean + go clean $(PKGS) + rm -f $(commands) deps: go get -t ./... -build: clean deps - go build +build: deps $(commands) -install: build - go install - -test: clean deps +test: deps go test -race ./... fmt: @@ -31,5 +32,5 @@ simplify: lint: @test -z "$(shell gofmt -l . | tee /dev/stderr)" || echo "[WARN] Fix formatting issues with 'make fmt'" - @golint -set_exit_status $${PKGS} - @go vet $${PKGS} + @golint -set_exit_status $(PKGS) + @go vet $(PKGS) diff --git a/cmd/client/list.go b/cmd/client/list.go index eda64bb..82f69d8 100644 --- a/cmd/client/list.go +++ b/cmd/client/list.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/google/subcommands" - "github.com/jhillyerd/inbucket/rest/client" + "github.com/jhillyerd/inbucket/pkg/rest/client" ) type listCmd struct { diff --git a/cmd/client/match.go b/cmd/client/match.go index df49a87..21a4ad0 100644 --- a/cmd/client/match.go +++ b/cmd/client/match.go @@ -10,7 +10,7 @@ import ( "time" "github.com/google/subcommands" - "github.com/jhillyerd/inbucket/rest/client" + "github.com/jhillyerd/inbucket/pkg/rest/client" ) type matchCmd struct { diff --git a/cmd/client/mbox.go b/cmd/client/mbox.go index cf2b46e..2105d24 100644 --- a/cmd/client/mbox.go +++ b/cmd/client/mbox.go @@ -7,7 +7,7 @@ import ( "os" "github.com/google/subcommands" - "github.com/jhillyerd/inbucket/rest/client" + "github.com/jhillyerd/inbucket/pkg/rest/client" ) type mboxCmd struct { diff --git a/inbucket.go b/cmd/inbucket/main.go similarity index 83% rename from inbucket.go rename to cmd/inbucket/main.go index 23c646a..0886a71 100644 --- a/inbucket.go +++ b/cmd/inbucket/main.go @@ -12,15 +12,15 @@ import ( "syscall" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/filestore" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/msghub" - "github.com/jhillyerd/inbucket/pop3d" - "github.com/jhillyerd/inbucket/rest" - "github.com/jhillyerd/inbucket/smtpd" - "github.com/jhillyerd/inbucket/webui" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/rest" + "github.com/jhillyerd/inbucket/pkg/server/pop3" + "github.com/jhillyerd/inbucket/pkg/server/smtp" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage/file" + "github.com/jhillyerd/inbucket/pkg/webui" ) var ( @@ -39,8 +39,8 @@ var ( shutdownChan = make(chan bool) // Server instances - smtpServer *smtpd.Server - pop3Server *pop3d.Server + smtpServer *smtp.Server + pop3Server *pop3.Server ) func init() { @@ -119,17 +119,17 @@ func main() { ds := filestore.DefaultFileDataStore() // Start HTTP server - httpd.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) - webui.SetupRoutes(httpd.Router) - rest.SetupRoutes(httpd.Router) - go httpd.Start(rootCtx) + web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) + webui.SetupRoutes(web.Router) + rest.SetupRoutes(web.Router) + go web.Start(rootCtx) // Start POP3 server - pop3Server = pop3d.New(config.GetPOP3Config(), shutdownChan, ds) + pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, ds) go pop3Server.Start(rootCtx) // Startup SMTP server - smtpServer = smtpd.NewServer(config.GetSMTPConfig(), shutdownChan, ds, msgHub) + smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, msgHub) go smtpServer.Start(rootCtx) // Loop forever waiting for signals or shutdown channel diff --git a/config/config.go b/pkg/config/config.go similarity index 100% rename from config/config.go rename to pkg/config/config.go diff --git a/log/logging.go b/pkg/log/logging.go similarity index 100% rename from log/logging.go rename to pkg/log/logging.go diff --git a/log/metrics.go b/pkg/log/metrics.go similarity index 100% rename from log/metrics.go rename to pkg/log/metrics.go diff --git a/log/stdout_unix.go b/pkg/log/stdout_unix.go similarity index 100% rename from log/stdout_unix.go rename to pkg/log/stdout_unix.go diff --git a/log/stdout_windows.go b/pkg/log/stdout_windows.go similarity index 100% rename from log/stdout_windows.go rename to pkg/log/stdout_windows.go diff --git a/msghub/hub.go b/pkg/msghub/hub.go similarity index 100% rename from msghub/hub.go rename to pkg/msghub/hub.go diff --git a/msghub/hub_test.go b/pkg/msghub/hub_test.go similarity index 100% rename from msghub/hub_test.go rename to pkg/msghub/hub_test.go diff --git a/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go similarity index 91% rename from rest/apiv1_controller.go rename to pkg/rest/apiv1_controller.go index 9037eb1..abd9b0b 100644 --- a/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -10,15 +10,15 @@ import ( "io/ioutil" "strconv" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/rest/model" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/rest/model" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // MailboxListV1 renders a list of messages in a mailbox -func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { @@ -48,11 +48,11 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) Size: msg.Size(), } } - return httpd.RenderJSON(w, jmessages) + return web.RenderJSON(w, jmessages) } // MailboxShowV1 renders a particular message from a mailbox -func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -96,7 +96,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) } } - return httpd.RenderJSON(w, + return web.RenderJSON(w, &model.JSONMessageV1{ Mailbox: name, ID: msg.ID(), @@ -115,7 +115,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) } // MailboxPurgeV1 deletes all messages from a mailbox -func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { @@ -133,11 +133,11 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context } log.Tracef("HTTP purged mailbox for %q", name) - return httpd.RenderJSON(w, "OK") + return web.RenderJSON(w, "OK") } // MailboxSourceV1 displays the raw source of a message, including headers. Renders text/plain -func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -171,7 +171,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Contex } // MailboxDeleteV1 removes a particular message from a mailbox -func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -197,5 +197,5 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *httpd.Contex return fmt.Errorf("Delete(%q) failed: %v", id, err) } - return httpd.RenderJSON(w, "OK") + return web.RenderJSON(w, "OK") } diff --git a/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go similarity index 99% rename from rest/apiv1_controller_test.go rename to pkg/rest/apiv1_controller_test.go index b1c9a00..7f5aba7 100644 --- a/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/jhillyerd/inbucket/datastore" + "github.com/jhillyerd/inbucket/pkg/storage" ) const ( diff --git a/rest/client/apiv1_client.go b/pkg/rest/client/apiv1_client.go similarity index 98% rename from rest/client/apiv1_client.go rename to pkg/rest/client/apiv1_client.go index d75bfa3..ad62dcb 100644 --- a/rest/client/apiv1_client.go +++ b/pkg/rest/client/apiv1_client.go @@ -8,7 +8,7 @@ import ( "net/url" "time" - "github.com/jhillyerd/inbucket/rest/model" + "github.com/jhillyerd/inbucket/pkg/rest/model" ) // Client accesses the Inbucket REST API v1 diff --git a/rest/client/apiv1_client_test.go b/pkg/rest/client/apiv1_client_test.go similarity index 100% rename from rest/client/apiv1_client_test.go rename to pkg/rest/client/apiv1_client_test.go diff --git a/rest/client/rest.go b/pkg/rest/client/rest.go similarity index 100% rename from rest/client/rest.go rename to pkg/rest/client/rest.go diff --git a/rest/client/rest_test.go b/pkg/rest/client/rest_test.go similarity index 100% rename from rest/client/rest_test.go rename to pkg/rest/client/rest_test.go diff --git a/rest/model/apiv1_model.go b/pkg/rest/model/apiv1_model.go similarity index 100% rename from rest/model/apiv1_model.go rename to pkg/rest/model/apiv1_model.go diff --git a/pkg/rest/routes.go b/pkg/rest/routes.go new file mode 100644 index 0000000..fad722d --- /dev/null +++ b/pkg/rest/routes.go @@ -0,0 +1,23 @@ +package rest + +import "github.com/gorilla/mux" +import "github.com/jhillyerd/inbucket/pkg/server/web" + +// SetupRoutes populates the routes for the REST interface +func SetupRoutes(r *mux.Router) { + // API v1 + r.Path("/api/v1/mailbox/{name}").Handler( + web.Handler(MailboxListV1)).Name("MailboxListV1").Methods("GET") + r.Path("/api/v1/mailbox/{name}").Handler( + web.Handler(MailboxPurgeV1)).Name("MailboxPurgeV1").Methods("DELETE") + r.Path("/api/v1/mailbox/{name}/{id}").Handler( + web.Handler(MailboxShowV1)).Name("MailboxShowV1").Methods("GET") + r.Path("/api/v1/mailbox/{name}/{id}").Handler( + web.Handler(MailboxDeleteV1)).Name("MailboxDeleteV1").Methods("DELETE") + r.Path("/api/v1/mailbox/{name}/{id}/source").Handler( + web.Handler(MailboxSourceV1)).Name("MailboxSourceV1").Methods("GET") + r.Path("/api/v1/monitor/messages").Handler( + web.Handler(MonitorAllMessagesV1)).Name("MonitorAllMessagesV1").Methods("GET") + r.Path("/api/v1/monitor/messages/{name}").Handler( + web.Handler(MonitorMailboxMessagesV1)).Name("MonitorMailboxMessagesV1").Methods("GET") +} diff --git a/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go similarity index 88% rename from rest/socketv1_controller.go rename to pkg/rest/socketv1_controller.go index 78bec2d..8bbf5a9 100644 --- a/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -5,11 +5,11 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/msghub" - "github.com/jhillyerd/inbucket/rest/model" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/rest/model" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) const ( @@ -145,16 +145,16 @@ func (ml *msgListener) Close() { } func MonitorAllMessagesV1( - w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { + w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Upgrade to Websocket conn, err := upgrader.Upgrade(w, req, nil) if err != nil { return err } - httpd.ExpWebSocketConnectsCurrent.Add(1) + web.ExpWebSocketConnectsCurrent.Add(1) defer func() { _ = conn.Close() - httpd.ExpWebSocketConnectsCurrent.Add(-1) + web.ExpWebSocketConnectsCurrent.Add(-1) }() log.Tracef("HTTP[%v] Upgraded to websocket", req.RemoteAddr) @@ -168,7 +168,7 @@ func MonitorAllMessagesV1( } func MonitorMailboxMessagesV1( - w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { + w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err @@ -178,10 +178,10 @@ func MonitorMailboxMessagesV1( if err != nil { return err } - httpd.ExpWebSocketConnectsCurrent.Add(1) + web.ExpWebSocketConnectsCurrent.Add(1) defer func() { _ = conn.Close() - httpd.ExpWebSocketConnectsCurrent.Add(-1) + web.ExpWebSocketConnectsCurrent.Add(-1) }() log.Tracef("HTTP[%v] Upgraded to websocket", req.RemoteAddr) diff --git a/rest/testutils_test.go b/pkg/rest/testutils_test.go similarity index 95% rename from rest/testutils_test.go rename to pkg/rest/testutils_test.go index 6ef4182..7955828 100644 --- a/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -10,10 +10,10 @@ import ( "time" "github.com/jhillyerd/enmime" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/msghub" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" ) type InputMessageData struct { @@ -184,7 +184,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { } w := httptest.NewRecorder() - httpd.Router.ServeHTTP(w, req) + web.Router.ServeHTTP(w, req) return w, nil } @@ -200,8 +200,8 @@ func setupWebServer(ds datastore.DataStore) *bytes.Buffer { PublicDir: "../themes/bootstrap/public", } shutdownChan := make(chan bool) - httpd.Initialize(cfg, shutdownChan, ds, &msghub.Hub{}) - SetupRoutes(httpd.Router) + web.Initialize(cfg, shutdownChan, ds, &msghub.Hub{}) + SetupRoutes(web.Router) return buf } diff --git a/pop3d/handler.go b/pkg/server/pop3/handler.go similarity index 99% rename from pop3d/handler.go rename to pkg/server/pop3/handler.go index 7f5967e..7922ca2 100644 --- a/pop3d/handler.go +++ b/pkg/server/pop3/handler.go @@ -1,4 +1,4 @@ -package pop3d +package pop3 import ( "bufio" @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" ) // State tracks the current mode of our POP3 state machine diff --git a/pop3d/listener.go b/pkg/server/pop3/listener.go similarity index 95% rename from pop3d/listener.go rename to pkg/server/pop3/listener.go index 58b0d82..42aaa56 100644 --- a/pop3d/listener.go +++ b/pkg/server/pop3/listener.go @@ -1,4 +1,4 @@ -package pop3d +package pop3 import ( "context" @@ -7,9 +7,9 @@ import ( "sync" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" ) // Server defines an instance of our POP3 server diff --git a/smtpd/handler.go b/pkg/server/smtp/handler.go similarity index 98% rename from smtpd/handler.go rename to pkg/server/smtp/handler.go index cbae977..96646b1 100644 --- a/smtpd/handler.go +++ b/pkg/server/smtp/handler.go @@ -1,4 +1,4 @@ -package smtpd +package smtp import ( "bufio" @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/msghub" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // State tracks the current mode of our SMTP state machine diff --git a/smtpd/handler_test.go b/pkg/server/smtp/handler_test.go similarity index 98% rename from smtpd/handler_test.go rename to pkg/server/smtp/handler_test.go index e541f4e..c1e6168 100644 --- a/smtpd/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -1,4 +1,4 @@ -package smtpd +package smtp import ( "bytes" @@ -13,9 +13,9 @@ import ( "testing" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/msghub" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/storage" ) type scriptStep struct { diff --git a/smtpd/listener.go b/pkg/server/smtp/listener.go similarity index 96% rename from smtpd/listener.go rename to pkg/server/smtp/listener.go index 60953b6..986d112 100644 --- a/smtpd/listener.go +++ b/pkg/server/smtp/listener.go @@ -1,4 +1,4 @@ -package smtpd +package smtp import ( "container/list" @@ -10,10 +10,10 @@ import ( "sync" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/msghub" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/storage" ) func init() { diff --git a/httpd/context.go b/pkg/server/web/context.go similarity index 90% rename from httpd/context.go rename to pkg/server/web/context.go index 6a54541..01d6c85 100644 --- a/httpd/context.go +++ b/pkg/server/web/context.go @@ -1,4 +1,4 @@ -package httpd +package web import ( "net/http" @@ -6,9 +6,9 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/sessions" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/msghub" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/storage" ) // Context is passed into every request handler function diff --git a/httpd/helpers.go b/pkg/server/web/helpers.go similarity index 97% rename from httpd/helpers.go rename to pkg/server/web/helpers.go index a5f66c6..c41e1e6 100644 --- a/httpd/helpers.go +++ b/pkg/server/web/helpers.go @@ -1,4 +1,4 @@ -package httpd +package web import ( "fmt" @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/log" ) // TemplateFuncs declares functions made available to all templates (including partials) diff --git a/httpd/helpers_test.go b/pkg/server/web/helpers_test.go similarity index 98% rename from httpd/helpers_test.go rename to pkg/server/web/helpers_test.go index fe0ca5c..39dc6eb 100644 --- a/httpd/helpers_test.go +++ b/pkg/server/web/helpers_test.go @@ -1,4 +1,4 @@ -package httpd +package web import ( "html/template" diff --git a/httpd/rest.go b/pkg/server/web/rest.go similarity index 96% rename from httpd/rest.go rename to pkg/server/web/rest.go index 805bbc9..aa9d864 100644 --- a/httpd/rest.go +++ b/pkg/server/web/rest.go @@ -1,4 +1,4 @@ -package httpd +package web import ( "encoding/json" diff --git a/httpd/server.go b/pkg/server/web/server.go similarity index 93% rename from httpd/server.go rename to pkg/server/web/server.go index f89b9ff..a9cd22a 100644 --- a/httpd/server.go +++ b/pkg/server/web/server.go @@ -1,5 +1,5 @@ -// Package httpd provides the plumbing for Inbucket's web GUI and RESTful API -package httpd +// Package web provides the plumbing for Inbucket's web GUI and RESTful API +package web import ( "context" @@ -12,10 +12,10 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/securecookie" "github.com/gorilla/sessions" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/msghub" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/storage" ) // Handler is a function type that handles an HTTP request in Inbucket diff --git a/httpd/template.go b/pkg/server/web/template.go similarity index 97% rename from httpd/template.go rename to pkg/server/web/template.go index 3aa81ce..216a1a4 100644 --- a/httpd/template.go +++ b/pkg/server/web/template.go @@ -1,4 +1,4 @@ -package httpd +package web import ( "html/template" @@ -8,7 +8,7 @@ import ( "strings" "sync" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/log" ) var cachedMutex sync.Mutex diff --git a/datastore/datastore.go b/pkg/storage/datastore.go similarity index 100% rename from datastore/datastore.go rename to pkg/storage/datastore.go diff --git a/filestore/fmessage.go b/pkg/storage/file/fmessage.go similarity index 98% rename from filestore/fmessage.go rename to pkg/storage/file/fmessage.go index 54da127..8d0fa3d 100644 --- a/filestore/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -11,8 +11,8 @@ import ( "time" "github.com/jhillyerd/enmime" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/log" ) // FileMessage implements Message and contains a little bit of data about a diff --git a/filestore/fstore.go b/pkg/storage/file/fstore.go similarity index 98% rename from filestore/fstore.go rename to pkg/storage/file/fstore.go index 7b15f27..98341d0 100644 --- a/filestore/fstore.go +++ b/pkg/storage/file/fstore.go @@ -11,10 +11,10 @@ import ( "sync" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // Name of index file in each mailbox diff --git a/filestore/fstore_test.go b/pkg/storage/file/fstore_test.go similarity index 99% rename from filestore/fstore_test.go rename to pkg/storage/file/fstore_test.go index 9cac985..b9fd41c 100644 --- a/filestore/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/jhillyerd/inbucket/config" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/stretchr/testify/assert" ) diff --git a/datastore/lock.go b/pkg/storage/lock.go similarity index 100% rename from datastore/lock.go rename to pkg/storage/lock.go diff --git a/datastore/lock_test.go b/pkg/storage/lock_test.go similarity index 96% rename from datastore/lock_test.go rename to pkg/storage/lock_test.go index 2da536a..203ebf1 100644 --- a/datastore/lock_test.go +++ b/pkg/storage/lock_test.go @@ -3,7 +3,7 @@ package datastore_test import ( "testing" - "github.com/jhillyerd/inbucket/datastore" + "github.com/jhillyerd/inbucket/pkg/storage" ) func TestHashLock(t *testing.T) { diff --git a/datastore/retention.go b/pkg/storage/retention.go similarity index 98% rename from datastore/retention.go rename to pkg/storage/retention.go index 7d52c27..ef3154b 100644 --- a/datastore/retention.go +++ b/pkg/storage/retention.go @@ -6,8 +6,8 @@ import ( "sync" "time" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/log" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/log" ) var ( diff --git a/datastore/retention_test.go b/pkg/storage/retention_test.go similarity index 100% rename from datastore/retention_test.go rename to pkg/storage/retention_test.go diff --git a/datastore/testing.go b/pkg/storage/testing.go similarity index 100% rename from datastore/testing.go rename to pkg/storage/testing.go diff --git a/stringutil/utils.go b/pkg/stringutil/utils.go similarity index 100% rename from stringutil/utils.go rename to pkg/stringutil/utils.go diff --git a/stringutil/utils_test.go b/pkg/stringutil/utils_test.go similarity index 100% rename from stringutil/utils_test.go rename to pkg/stringutil/utils_test.go diff --git a/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go similarity index 84% rename from webui/mailbox_controller.go rename to pkg/webui/mailbox_controller.go index 7f2d6e8..bc2ef77 100644 --- a/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -7,29 +7,29 @@ import ( "net/http" "strconv" - "github.com/jhillyerd/inbucket/datastore" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/log" - "github.com/jhillyerd/inbucket/sanitize" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/stringutil" + "github.com/jhillyerd/inbucket/pkg/webui/sanitize" ) // MailboxIndex renders the index page for a particular mailbox -func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Form values must be validated manually name := req.FormValue("name") selected := req.FormValue("id") if len(name) == 0 { ctx.Session.AddFlash("Account name is required", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } name, err = stringutil.ParseMailboxName(name) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } // Remember this mailbox was visited @@ -40,7 +40,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) return err } // Render template - return httpd.RenderTemplate("mailbox/index.html", w, map[string]interface{}{ + return web.RenderTemplate("mailbox/index.html", w, map[string]interface{}{ "ctx": ctx, "errorFlash": errorFlash, "name": name, @@ -49,24 +49,24 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) } // MailboxLink handles pretty links to a particular message. Renders a redirect -func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } // Build redirect - uri := fmt.Sprintf("%s?name=%s&id=%s", httpd.Reverse("MailboxIndex"), name, id) + uri := fmt.Sprintf("%s?name=%s&id=%s", web.Reverse("MailboxIndex"), name, id) http.Redirect(w, req, uri, http.StatusSeeOther) return nil } // MailboxList renders a list of messages in a mailbox. Renders a partial -func MailboxList(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { @@ -84,7 +84,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } log.Tracef("Got %v messsages", len(messages)) // Render partial template - return httpd.RenderPartial("mailbox/_list.html", w, map[string]interface{}{ + return web.RenderPartial("mailbox/_list.html", w, map[string]interface{}{ "ctx": ctx, "name": name, "messages": messages, @@ -92,7 +92,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } // MailboxShow renders a particular message from a mailbox. Renders an HTML partial -func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -117,7 +117,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( if err != nil { return fmt.Errorf("ReadBody(%q) failed: %v", id, err) } - body := template.HTML(httpd.TextToHTML(mime.Text)) + body := template.HTML(web.TextToHTML(mime.Text)) htmlAvailable := mime.HTML != "" var htmlBody template.HTML if htmlAvailable { @@ -128,7 +128,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } } // Render partial template - return httpd.RenderPartial("mailbox/_show.html", w, map[string]interface{}{ + return web.RenderPartial("mailbox/_show.html", w, map[string]interface{}{ "ctx": ctx, "name": name, "message": msg, @@ -141,7 +141,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } // MailboxHTML displays the HTML content of a message. Renders a partial -func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -168,7 +168,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } // Render partial template w.Header().Set("Content-Type", "text/html; charset=UTF-8") - return httpd.RenderPartial("mailbox/_html.html", w, map[string]interface{}{ + return web.RenderPartial("mailbox/_html.html", w, map[string]interface{}{ "ctx": ctx, "name": name, "message": message, @@ -178,7 +178,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( } // MailboxSource displays the raw source of a message, including headers. Renders text/plain -func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) @@ -213,14 +213,14 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) // MailboxDownloadAttach sends the attachment to the client; disposition: // attachment, type: application/octet-stream -func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } numStr := ctx.Vars["num"] @@ -228,7 +228,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *httpd. if err != nil { ctx.Session.AddFlash("Attachment number must be unsigned numeric", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } mb, err := ctx.DataStore.MailboxFor(name) @@ -252,7 +252,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *httpd. if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } part := body.Attachments[num] @@ -266,13 +266,13 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *httpd. } // MailboxViewAttach sends the attachment to the client for online viewing -func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } id := ctx.Vars["id"] @@ -281,7 +281,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *httpd.Cont if err != nil { ctx.Session.AddFlash("Attachment number must be unsigned numeric", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } mb, err := ctx.DataStore.MailboxFor(name) @@ -305,7 +305,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *httpd.Cont if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } part := body.Attachments[num] diff --git a/webui/recent.go b/pkg/webui/recent.go similarity index 83% rename from webui/recent.go rename to pkg/webui/recent.go index 14118ac..97168d3 100644 --- a/webui/recent.go +++ b/pkg/webui/recent.go @@ -1,7 +1,7 @@ package webui import ( - "github.com/jhillyerd/inbucket/httpd" + "github.com/jhillyerd/inbucket/pkg/server/web" ) const ( @@ -12,7 +12,7 @@ const ( ) // RememberMailbox manages the list of recently accessed mailboxes stored in the session -func RememberMailbox(ctx *httpd.Context, mailbox string) { +func RememberMailbox(ctx *web.Context, mailbox string) { recent := RecentMailboxes(ctx) newRecent := make([]string, 1, maxRemembered) newRecent[0] = mailbox @@ -28,7 +28,7 @@ func RememberMailbox(ctx *httpd.Context, mailbox string) { } // RecentMailboxes returns a slice of the most recently accessed mailboxes -func RecentMailboxes(ctx *httpd.Context) []string { +func RecentMailboxes(ctx *web.Context) []string { val := ctx.Session.Values[mailboxKey] recent, _ := val.([]string) return recent diff --git a/webui/root_controller.go b/pkg/webui/root_controller.go similarity index 75% rename from webui/root_controller.go rename to pkg/webui/root_controller.go index 9a64bc5..85d1662 100644 --- a/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -6,13 +6,13 @@ import ( "io/ioutil" "net/http" - "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/httpd" - "github.com/jhillyerd/inbucket/stringutil" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // RootIndex serves the Inbucket landing page -func RootIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func RootIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { greeting, err := ioutil.ReadFile(config.GetWebConfig().GreetingFile) if err != nil { return fmt.Errorf("Failed to load greeting: %v", err) @@ -23,7 +23,7 @@ func RootIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (er return err } // Render template - return httpd.RenderTemplate("root/index.html", w, map[string]interface{}{ + return web.RenderTemplate("root/index.html", w, map[string]interface{}{ "ctx": ctx, "errorFlash": errorFlash, "greeting": template.HTML(string(greeting)), @@ -31,11 +31,11 @@ func RootIndex(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (er } // RootMonitor serves the Inbucket monitor page -func RootMonitor(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func RootMonitor(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { if !config.GetWebConfig().MonitorVisible { ctx.Session.AddFlash("Monitor is disabled in configuration", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } // Get flash messages, save session @@ -44,25 +44,25 @@ func RootMonitor(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) ( return err } // Render template - return httpd.RenderTemplate("root/monitor.html", w, map[string]interface{}{ + return web.RenderTemplate("root/monitor.html", w, map[string]interface{}{ "ctx": ctx, "errorFlash": errorFlash, }) } // RootMonitorMailbox serves the Inbucket monitor page for a particular mailbox -func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { if !config.GetWebConfig().MonitorVisible { ctx.Session.AddFlash("Monitor is disabled in configuration", "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) - http.Redirect(w, req, httpd.Reverse("RootIndex"), http.StatusSeeOther) + http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } // Get flash messages, save session @@ -71,7 +71,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *httpd.Con return err } // Render template - return httpd.RenderTemplate("root/monitor.html", w, map[string]interface{}{ + return web.RenderTemplate("root/monitor.html", w, map[string]interface{}{ "ctx": ctx, "errorFlash": errorFlash, "name": name, @@ -79,7 +79,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *httpd.Con } // RootStatus serves the Inbucket status page -func RootStatus(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (err error) { +func RootStatus(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { smtpListener := fmt.Sprintf("%s:%d", config.GetSMTPConfig().IP4address.String(), config.GetSMTPConfig().IP4port) pop3Listener := fmt.Sprintf("%s:%d", config.GetPOP3Config().IP4address.String(), @@ -92,7 +92,7 @@ func RootStatus(w http.ResponseWriter, req *http.Request, ctx *httpd.Context) (e return err } // Render template - return httpd.RenderTemplate("root/status.html", w, map[string]interface{}{ + return web.RenderTemplate("root/status.html", w, map[string]interface{}{ "ctx": ctx, "errorFlash": errorFlash, "version": config.Version, diff --git a/pkg/webui/routes.go b/pkg/webui/routes.go new file mode 100644 index 0000000..1bbe888 --- /dev/null +++ b/pkg/webui/routes.go @@ -0,0 +1,35 @@ +// Package webui powers Inbucket's web GUI +package webui + +import ( + "github.com/gorilla/mux" + "github.com/jhillyerd/inbucket/pkg/server/web" +) + +// SetupRoutes populates routes for the webui into the provided Router +func SetupRoutes(r *mux.Router) { + r.Path("/").Handler( + web.Handler(RootIndex)).Name("RootIndex").Methods("GET") + r.Path("/monitor").Handler( + web.Handler(RootMonitor)).Name("RootMonitor").Methods("GET") + r.Path("/monitor/{name}").Handler( + web.Handler(RootMonitorMailbox)).Name("RootMonitorMailbox").Methods("GET") + r.Path("/status").Handler( + web.Handler(RootStatus)).Name("RootStatus").Methods("GET") + r.Path("/link/{name}/{id}").Handler( + web.Handler(MailboxLink)).Name("MailboxLink").Methods("GET") + r.Path("/mailbox").Handler( + web.Handler(MailboxIndex)).Name("MailboxIndex").Methods("GET") + r.Path("/mailbox/{name}").Handler( + web.Handler(MailboxList)).Name("MailboxList").Methods("GET") + r.Path("/mailbox/{name}/{id}").Handler( + web.Handler(MailboxShow)).Name("MailboxShow").Methods("GET") + r.Path("/mailbox/{name}/{id}/html").Handler( + web.Handler(MailboxHTML)).Name("MailboxHtml").Methods("GET") + r.Path("/mailbox/{name}/{id}/source").Handler( + web.Handler(MailboxSource)).Name("MailboxSource").Methods("GET") + r.Path("/mailbox/dattach/{name}/{id}/{num}/{file}").Handler( + web.Handler(MailboxDownloadAttach)).Name("MailboxDownloadAttach").Methods("GET") + r.Path("/mailbox/vattach/{name}/{id}/{num}/{file}").Handler( + web.Handler(MailboxViewAttach)).Name("MailboxViewAttach").Methods("GET") +} diff --git a/sanitize/css.go b/pkg/webui/sanitize/css.go similarity index 100% rename from sanitize/css.go rename to pkg/webui/sanitize/css.go diff --git a/sanitize/css_test.go b/pkg/webui/sanitize/css_test.go similarity index 100% rename from sanitize/css_test.go rename to pkg/webui/sanitize/css_test.go diff --git a/sanitize/html.go b/pkg/webui/sanitize/html.go similarity index 100% rename from sanitize/html.go rename to pkg/webui/sanitize/html.go diff --git a/sanitize/html_test.go b/pkg/webui/sanitize/html_test.go similarity index 98% rename from sanitize/html_test.go rename to pkg/webui/sanitize/html_test.go index c6acf09..c5443f4 100644 --- a/sanitize/html_test.go +++ b/pkg/webui/sanitize/html_test.go @@ -3,7 +3,7 @@ package sanitize_test import ( "testing" - "github.com/jhillyerd/inbucket/sanitize" + "github.com/jhillyerd/inbucket/pkg/webui/sanitize" ) // TestHTMLPlainStrings test plain text passthrough diff --git a/rest/routes.go b/rest/routes.go deleted file mode 100644 index 1bc73d1..0000000 --- a/rest/routes.go +++ /dev/null @@ -1,23 +0,0 @@ -package rest - -import "github.com/gorilla/mux" -import "github.com/jhillyerd/inbucket/httpd" - -// SetupRoutes populates the routes for the REST interface -func SetupRoutes(r *mux.Router) { - // API v1 - r.Path("/api/v1/mailbox/{name}").Handler( - httpd.Handler(MailboxListV1)).Name("MailboxListV1").Methods("GET") - r.Path("/api/v1/mailbox/{name}").Handler( - httpd.Handler(MailboxPurgeV1)).Name("MailboxPurgeV1").Methods("DELETE") - r.Path("/api/v1/mailbox/{name}/{id}").Handler( - httpd.Handler(MailboxShowV1)).Name("MailboxShowV1").Methods("GET") - r.Path("/api/v1/mailbox/{name}/{id}").Handler( - httpd.Handler(MailboxDeleteV1)).Name("MailboxDeleteV1").Methods("DELETE") - r.Path("/api/v1/mailbox/{name}/{id}/source").Handler( - httpd.Handler(MailboxSourceV1)).Name("MailboxSourceV1").Methods("GET") - r.Path("/api/v1/monitor/messages").Handler( - httpd.Handler(MonitorAllMessagesV1)).Name("MonitorAllMessagesV1").Methods("GET") - r.Path("/api/v1/monitor/messages/{name}").Handler( - httpd.Handler(MonitorMailboxMessagesV1)).Name("MonitorMailboxMessagesV1").Methods("GET") -} diff --git a/webui/routes.go b/webui/routes.go deleted file mode 100644 index 3103d7f..0000000 --- a/webui/routes.go +++ /dev/null @@ -1,35 +0,0 @@ -// Package webui powers Inbucket's web GUI -package webui - -import ( - "github.com/gorilla/mux" - "github.com/jhillyerd/inbucket/httpd" -) - -// SetupRoutes populates routes for the webui into the provided Router -func SetupRoutes(r *mux.Router) { - r.Path("/").Handler( - httpd.Handler(RootIndex)).Name("RootIndex").Methods("GET") - r.Path("/monitor").Handler( - httpd.Handler(RootMonitor)).Name("RootMonitor").Methods("GET") - r.Path("/monitor/{name}").Handler( - httpd.Handler(RootMonitorMailbox)).Name("RootMonitorMailbox").Methods("GET") - r.Path("/status").Handler( - httpd.Handler(RootStatus)).Name("RootStatus").Methods("GET") - r.Path("/link/{name}/{id}").Handler( - httpd.Handler(MailboxLink)).Name("MailboxLink").Methods("GET") - r.Path("/mailbox").Handler( - httpd.Handler(MailboxIndex)).Name("MailboxIndex").Methods("GET") - r.Path("/mailbox/{name}").Handler( - httpd.Handler(MailboxList)).Name("MailboxList").Methods("GET") - r.Path("/mailbox/{name}/{id}").Handler( - httpd.Handler(MailboxShow)).Name("MailboxShow").Methods("GET") - r.Path("/mailbox/{name}/{id}/html").Handler( - httpd.Handler(MailboxHTML)).Name("MailboxHtml").Methods("GET") - r.Path("/mailbox/{name}/{id}/source").Handler( - httpd.Handler(MailboxSource)).Name("MailboxSource").Methods("GET") - r.Path("/mailbox/dattach/{name}/{id}/{num}/{file}").Handler( - httpd.Handler(MailboxDownloadAttach)).Name("MailboxDownloadAttach").Methods("GET") - r.Path("/mailbox/vattach/{name}/{id}/{num}/{file}").Handler( - httpd.Handler(MailboxViewAttach)).Name("MailboxViewAttach").Methods("GET") -} From 94167fa3133c26b91805f17f12f4b9d74709cf37 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Fri, 9 Mar 2018 22:01:43 -0800 Subject: [PATCH 02/82] Resolve linter errors exposed by fixed Makefile - TravisCI didn't like "POSIX" ::= syntax --- .travis.yml | 3 +-- Makefile | 6 +++--- pkg/log/logging.go | 2 +- pkg/rest/model/apiv1_model.go | 1 + pkg/rest/socketv1_controller.go | 4 ++++ pkg/storage/file/fmessage.go | 2 +- pkg/storage/file/fstore.go | 3 ++- pkg/storage/lock.go | 4 ++++ pkg/storage/testing.go | 1 + pkg/webui/sanitize/html.go | 1 + 10 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9ea160..c820ee3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,12 @@ language: go sudo: false env: - - DEPLOY_WITH_MAJOR="1.9" + - DEPLOY_WITH_MAJOR="1.10" before_script: - go get github.com/golang/lint/golint go: - - 1.9.x - "1.10" deploy: diff --git a/Makefile b/Makefile index ae01ee8..c5104cc 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ SHELL = /bin/sh -SRC ::= $(shell find . -type f -name '*.go' -not -path "./vendor/*") -PKGS ::= $(shell go list ./... | grep -v /vendor/) +SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*") +PKGS := $(shell go list ./... | grep -v /vendor/) .PHONY: all build clean fmt lint simplify test -commands ::= client inbucket +commands = client inbucket all: clean test lint build diff --git a/pkg/log/logging.go b/pkg/log/logging.go index 89ab7e7..7a73e24 100644 --- a/pkg/log/logging.go +++ b/pkg/log/logging.go @@ -128,7 +128,7 @@ func openLogFile() error { var err error logf, err = os.OpenFile(logfname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) if err != nil { - return fmt.Errorf("Failed to create %v: %v\n", logfname, err) + return fmt.Errorf("failed to create %v: %v", logfname, err) } golog.SetOutput(logf) Tracef("Opened new logfile") diff --git a/pkg/rest/model/apiv1_model.go b/pkg/rest/model/apiv1_model.go index f2d3e76..1e4f7f0 100644 --- a/pkg/rest/model/apiv1_model.go +++ b/pkg/rest/model/apiv1_model.go @@ -30,6 +30,7 @@ type JSONMessageV1 struct { Attachments []*JSONMessageAttachmentV1 `json:"attachments"` } +// JSONMessageAttachmentV1 contains information about a MIME attachment type JSONMessageAttachmentV1 struct { FileName string `json:"filename"` ContentType string `json:"content-type"` diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index 8bbf5a9..5ad2dbf 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -144,6 +144,8 @@ func (ml *msgListener) Close() { } } +// MonitorAllMessagesV1 is a web handler which upgrades the connection to a websocket and notifies +// the client of all messages received. func MonitorAllMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Upgrade to Websocket @@ -167,6 +169,8 @@ func MonitorAllMessagesV1( return nil } +// MonitorMailboxMessagesV1 is a web handler which upgrades the connection to a websocket and +// notifies the client of messages received by a particular mailbox. func MonitorMailboxMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 8d0fa3d..22de241 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -11,8 +11,8 @@ import ( "time" "github.com/jhillyerd/enmime" - "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" ) // FileMessage implements Message and contains a little bit of data about a diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 98341d0..7d7dd35 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -12,8 +12,8 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -140,6 +140,7 @@ func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { return mailboxes, nil } +// LockFor returns the RWMutex for this mailbox, or an error. func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { diff --git a/pkg/storage/lock.go b/pkg/storage/lock.go index a08dacc..5247702 100644 --- a/pkg/storage/lock.go +++ b/pkg/storage/lock.go @@ -5,8 +5,12 @@ import ( "sync" ) +// HashLock holds a fixed length array of mutexes. This approach allows concurrent mailbox +// access in most cases without requiring an infinite number of mutexes. type HashLock [4096]sync.RWMutex +// Get returns a RWMutex based on the first 12 bits of the mailbox hash. Hash must be a hexidecimal +// string of three or more characters. func (h *HashLock) Get(hash string) *sync.RWMutex { if len(hash) < 3 { return nil diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 8e0799c..aa6c3de 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -27,6 +27,7 @@ func (m *MockDataStore) AllMailboxes() ([]Mailbox, error) { return args.Get(0).([]Mailbox), args.Error(1) } +// LockFor mock function returns a new RWMutex, never errors. func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil } diff --git a/pkg/webui/sanitize/html.go b/pkg/webui/sanitize/html.go index 475880f..cd7a7ae 100644 --- a/pkg/webui/sanitize/html.go +++ b/pkg/webui/sanitize/html.go @@ -18,6 +18,7 @@ var ( AllowAttrs("style").Matching(cssSafe).Globally() ) +// HTML sanitizes the provided html, while attempting to preserve inline CSS styling. func HTML(html string) (output string, err error) { output, err = sanitizeStyleTags(html) if err != nil { From 0016c6d5df138a743c2bccf6cac185ffab6d5996 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 12:15:05 -0800 Subject: [PATCH 03/82] readme: updated for reorg, #79 - noted Homebrew is broken --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 287d6cc..a76c3eb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ to contribute code to the project check out [CONTRIBUTING.md]. ## Homebrew Tap +(currently broken, being tracked in [issue +#68](https://github.com/jhillyerd/inbucket/issues/68)) + Inbucket has an OS X [Homebrew] tap available as [jhillyerd/inbucket][Homebrew Tap], see the `README.md` there for installation instructions. @@ -31,7 +34,7 @@ You will need a functioning [Go installation][Google Go] for this to work. Grab the Inbucket source code and compile the daemon: - go get -v github.com/jhillyerd/inbucket + go get -v github.com/jhillyerd/inbucket/cmd/inbucket Edit etc/inbucket.conf and tailor to your environment. It should work on most Unix and OS X machines as is. Launch the daemon: From 1f56e06fb9897cabca85d9229a70e3ccfca928c3 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 12:45:56 -0800 Subject: [PATCH 04/82] docker: fix build for #79 - Build with Go 1.10 - install.sh: git fetch fails with ssh remotes, removed --- Dockerfile | 2 +- etc/docker/install.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index cebab97..ba393f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Docker build file for Inbucket, see https://www.docker.io/ # Inbucket website: http://www.inbucket.org/ -FROM golang:1.9-alpine +FROM golang:1.10-alpine MAINTAINER James Hillyerd, @jameshillyerd # Configuration (WORKDIR doesn't support env vars) diff --git a/etc/docker/install.sh b/etc/docker/install.sh index 916c745..54507db 100755 --- a/etc/docker/install.sh +++ b/etc/docker/install.sh @@ -16,8 +16,6 @@ apk add --no-cache --virtual .build-deps git # Setup export GOBIN="$bindir" cd "$srcdir" -# Fetch tags for describe -git fetch -t builddate="$(date -Iseconds)" buildver="$(git describe --tags --always)" @@ -30,7 +28,7 @@ echo "### Testing Inbucket" go test ./... echo "### Building Inbucket" -go build -o inbucket -ldflags "-X 'main.version=$buildver' -X 'main.date=$builddate'" -v . +go build -o inbucket -ldflags "-X 'main.version=$buildver' -X 'main.date=$builddate'" -v ./cmd/inbucket echo "### Installing Inbucket" set -x From a58dfc5e4fb92b8106ab9e8de10fd224ce3e31e9 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 13:34:35 -0800 Subject: [PATCH 05/82] storage: finish renaming storage packages for #79 #69 - storage: rename DataStore to Store - file: rename types to appease linter --- cmd/inbucket/main.go | 2 +- pkg/rest/apiv1_controller.go | 6 +-- pkg/rest/apiv1_controller_test.go | 30 +++++------ pkg/rest/testutils_test.go | 6 +-- pkg/server/pop3/handler.go | 28 +++++------ pkg/server/pop3/listener.go | 4 +- pkg/server/smtp/handler.go | 2 +- pkg/server/smtp/handler_test.go | 18 +++---- pkg/server/smtp/listener.go | 12 ++--- pkg/server/web/context.go | 2 +- pkg/server/web/server.go | 4 +- pkg/storage/file/fmessage.go | 44 ++++++++-------- pkg/storage/file/fstore.go | 64 ++++++++++++------------ pkg/storage/file/fstore_test.go | 14 +++--- pkg/storage/lock.go | 2 +- pkg/storage/lock_test.go | 4 +- pkg/storage/retention.go | 6 +-- pkg/storage/retention_test.go | 2 +- pkg/storage/{datastore.go => storage.go} | 8 +-- pkg/storage/testing.go | 2 +- pkg/webui/mailbox_controller.go | 10 ++-- 21 files changed, 135 insertions(+), 135 deletions(-) rename pkg/storage/{datastore.go => storage.go} (87%) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 0886a71..6e25e44 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -116,7 +116,7 @@ func main() { msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) // Grab our datastore - ds := filestore.DefaultFileDataStore() + ds := file.DefaultStore() // Start HTTP server web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index abd9b0b..010bfac 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -65,7 +65,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } msg, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -150,7 +150,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -184,7 +184,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 7f5aba7..9c4e789 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -31,7 +31,7 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := &datastore.MockDataStore{} + ds := &storage.MockDataStore{} logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -45,9 +45,9 @@ func TestRestMailboxList(t *testing.T) { } // Test empty mailbox - emptybox := &datastore.MockMailbox{} + emptybox := &storage.MockMailbox{} ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessages").Return([]datastore.Message{}, nil) + emptybox.On("GetMessages").Return([]storage.Message{}, nil) w, err = testRestGet(baseURL + "/mailbox/empty") expectCode = 200 @@ -59,7 +59,7 @@ func TestRestMailboxList(t *testing.T) { } // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&datastore.MockMailbox{}, fmt.Errorf("Internal error")) + ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) w, err = testRestGet(baseURL + "/mailbox/error") expectCode = 500 if err != nil { @@ -77,9 +77,9 @@ func TestRestMailboxList(t *testing.T) { } // Test MailboxFor error - error2box := &datastore.MockMailbox{} + error2box := &storage.MockMailbox{} ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessages").Return([]datastore.Message{}, fmt.Errorf("Internal error 2")) + error2box.On("GetMessages").Return([]storage.Message{}, fmt.Errorf("Internal error 2")) w, err = testRestGet(baseURL + "/mailbox/error2") expectCode = 500 @@ -107,11 +107,11 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - goodbox := &datastore.MockMailbox{} + goodbox := &storage.MockMailbox{} ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() msg2 := data2.MockMessage() - goodbox.On("GetMessages").Return([]datastore.Message{msg1, msg2}, nil) + goodbox.On("GetMessages").Return([]storage.Message{msg1, msg2}, nil) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -155,7 +155,7 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := &datastore.MockDataStore{} + ds := &storage.MockDataStore{} logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -169,9 +169,9 @@ func TestRestMessage(t *testing.T) { } // Test requesting a message that does not exist - emptybox := &datastore.MockMailbox{} + emptybox := &storage.MockMailbox{} ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessage", "0001").Return(&datastore.MockMessage{}, datastore.ErrNotExist) + emptybox.On("GetMessage", "0001").Return(&storage.MockMessage{}, storage.ErrNotExist) w, err = testRestGet(baseURL + "/mailbox/empty/0001") expectCode = 404 @@ -183,7 +183,7 @@ func TestRestMessage(t *testing.T) { } // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&datastore.MockMailbox{}, fmt.Errorf("Internal error")) + ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) w, err = testRestGet(baseURL + "/mailbox/error/0001") expectCode = 500 if err != nil { @@ -201,9 +201,9 @@ func TestRestMessage(t *testing.T) { } // Test GetMessage error - error2box := &datastore.MockMailbox{} + error2box := &storage.MockMailbox{} ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessage", "0001").Return(&datastore.MockMessage{}, fmt.Errorf("Internal error 2")) + error2box.On("GetMessage", "0001").Return(&storage.MockMessage{}, fmt.Errorf("Internal error 2")) w, err = testRestGet(baseURL + "/mailbox/error2/0001") expectCode = 500 @@ -228,7 +228,7 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - goodbox := &datastore.MockMailbox{} + goodbox := &storage.MockMailbox{} ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() goodbox.On("GetMessage", "0001").Return(msg1, nil) diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 7955828..ef294df 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -25,8 +25,8 @@ type InputMessageData struct { HTML, Text string } -func (d *InputMessageData) MockMessage() *datastore.MockMessage { - msg := &datastore.MockMessage{} +func (d *InputMessageData) MockMessage() *storage.MockMessage { + msg := &storage.MockMessage{} msg.On("ID").Return(d.ID) msg.On("From").Return(d.From) msg.On("To").Return(d.To) @@ -188,7 +188,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(ds datastore.DataStore) *bytes.Buffer { +func setupWebServer(ds storage.Store) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 7922ca2..98cce65 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,18 +57,18 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - mailbox datastore.Mailbox // Mailbox instance - messages []datastore.Message // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + mailbox storage.Mailbox // Mailbox instance + messages []storage.Message // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -432,7 +432,7 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg datastore.Message) { +func (ses *Session) sendMessage(msg storage.Message) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -465,7 +465,7 @@ func (ses *Session) sendMessage(msg datastore.Message) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg datastore.Message, lineCount int) { +func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index 42aaa56..c971854 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -17,14 +17,14 @@ type Server struct { host string domain string maxIdleSeconds int - dataStore datastore.DataStore + dataStore storage.Store listener net.Listener globalShutdown chan bool waitgroup *sync.WaitGroup } // New creates a new Server struct -func New(cfg config.POP3Config, shutdownChan chan bool, ds datastore.DataStore) *Server { +func New(cfg config.POP3Config, shutdownChan chan bool, ds storage.Store) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), domain: cfg.Domain, diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 96646b1..a678d3a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -73,7 +73,7 @@ var commands = map[string]bool{ // recipientDetails for message delivery type recipientDetails struct { address, localPart, domainPart string - mailbox datastore.Mailbox + mailbox storage.Mailbox } // Session holds the state of an SMTP session diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index c1e6168..b53b651 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -26,7 +26,7 @@ type scriptStep struct { // Test commands in GREET state func TestGreetState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} + mds := &storage.MockDataStore{} server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -83,7 +83,7 @@ func TestGreetState(t *testing.T) { // Test commands in READY state func TestReadyState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} + mds := &storage.MockDataStore{} server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -144,9 +144,9 @@ func TestReadyState(t *testing.T) { // Test commands in MAIL state func TestMailState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} - mb1 := &datastore.MockMailbox{} - msg1 := &datastore.MockMessage{} + mds := &storage.MockDataStore{} + mb1 := &storage.MockMailbox{} + msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) mb1.On("NewMessage").Return(msg1, nil) mb1.On("Name").Return("u1") @@ -259,9 +259,9 @@ func TestMailState(t *testing.T) { // Test commands in DATA state func TestDataState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} - mb1 := &datastore.MockMailbox{} - msg1 := &datastore.MockMessage{} + mds := &storage.MockDataStore{} + mb1 := &storage.MockMailbox{} + msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) mb1.On("NewMessage").Return(msg1, nil) mb1.On("Name").Return("u1") @@ -367,7 +367,7 @@ func (m *mockConn) SetDeadline(t time.Time) error { return nil } func (m *mockConn) SetReadDeadline(t time.Time) error { return nil } func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } -func setupSMTPServer(ds datastore.DataStore) (s *Server, buf *bytes.Buffer, teardown func()) { +func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown func()) { // Test Server Config cfg := config.SMTPConfig{ IP4address: net.IPv4(127, 0, 0, 1), diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 986d112..4586174 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -48,10 +48,10 @@ type Server struct { storeMessages bool // Dependencies - dataStore datastore.DataStore // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info - retentionScanner *datastore.RetentionScanner // Deletes expired messages + dataStore storage.Store // Mailbox/message store + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info + retentionScanner *storage.RetentionScanner // Deletes expired messages // State listener net.Listener // Incoming network connections @@ -83,7 +83,7 @@ var ( func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, - ds datastore.DataStore, + ds storage.Store, msgHub *msghub.Hub) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), @@ -96,7 +96,7 @@ func NewServer( globalShutdown: globalShutdown, dataStore: ds, msgHub: msgHub, - retentionScanner: datastore.NewRetentionScanner(ds, globalShutdown), + retentionScanner: storage.NewRetentionScanner(ds, globalShutdown), waitgroup: new(sync.WaitGroup), } } diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index 01d6c85..422fdb5 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -15,7 +15,7 @@ import ( type Context struct { Vars map[string]string Session *sessions.Session - DataStore datastore.DataStore + DataStore storage.Store MsgHub *msghub.Hub WebConfig config.WebConfig IsJSON bool diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index a9cd22a..5c82430 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -23,7 +23,7 @@ type Handler func(http.ResponseWriter, *http.Request, *Context) error var ( // DataStore is where all the mailboxes and messages live - DataStore datastore.DataStore + DataStore storage.Store // msgHub holds a reference to the message pub/sub system msgHub *msghub.Hub @@ -51,7 +51,7 @@ func init() { func Initialize( cfg config.WebConfig, shutdownChan chan bool, - ds datastore.DataStore, + ds storage.Store, mh *msghub.Hub) { webConfig = cfg diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 22de241..4e496f8 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bufio" @@ -15,10 +15,10 @@ import ( "github.com/jhillyerd/inbucket/pkg/storage" ) -// FileMessage implements Message and contains a little bit of data about a +// Message implements Message and contains a little bit of data about a // particular email message, and methods to retrieve the rest of it from disk. -type FileMessage struct { - mailbox *FileMailbox +type Message struct { + mailbox *Mailbox // Stored in GOB Fid string Fdate time.Time @@ -34,7 +34,7 @@ type FileMessage struct { // NewMessage creates a new FileMessage object and sets the Date and Id fields. // It will also delete messages over messageCap if configured. -func (mb *FileMailbox) NewMessage() (datastore.Message, error) { +func (mb *Mailbox) NewMessage() (storage.Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -54,50 +54,50 @@ func (mb *FileMailbox) NewMessage() (datastore.Message, error) { date := time.Now() id := generateID(date) - return &FileMessage{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil + return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil } // ID gets the ID of the Message -func (m *FileMessage) ID() string { +func (m *Message) ID() string { return m.Fid } // Date returns the date/time this Message was received by Inbucket -func (m *FileMessage) Date() time.Time { +func (m *Message) Date() time.Time { return m.Fdate } // From returns the value of the Message From header -func (m *FileMessage) From() string { +func (m *Message) From() string { return m.Ffrom } // To returns the value of the Message To header -func (m *FileMessage) To() []string { +func (m *Message) To() []string { return m.Fto } // Subject returns the value of the Message Subject header -func (m *FileMessage) Subject() string { +func (m *Message) Subject() string { return m.Fsubject } // String returns a string in the form: "Subject()" from From() -func (m *FileMessage) String() string { +func (m *Message) String() string { return fmt.Sprintf("\"%v\" from %v", m.Fsubject, m.Ffrom) } // Size returns the size of the Message on disk in bytes -func (m *FileMessage) Size() int64 { +func (m *Message) Size() int64 { return m.Fsize } -func (m *FileMessage) rawPath() string { +func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } // ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object -func (m *FileMessage) ReadHeader() (msg *mail.Message, err error) { +func (m *Message) ReadHeader() (msg *mail.Message, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -113,7 +113,7 @@ func (m *FileMessage) ReadHeader() (msg *mail.Message, err error) { } // ReadBody opens the .raw portion of a Message and returns a MIMEBody object -func (m *FileMessage) ReadBody() (body *enmime.Envelope, err error) { +func (m *Message) ReadBody() (body *enmime.Envelope, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -133,7 +133,7 @@ func (m *FileMessage) ReadBody() (body *enmime.Envelope, err error) { } // RawReader opens the .raw portion of a Message as an io.ReadCloser -func (m *FileMessage) RawReader() (reader io.ReadCloser, err error) { +func (m *Message) RawReader() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -142,7 +142,7 @@ func (m *FileMessage) RawReader() (reader io.ReadCloser, err error) { } // ReadRaw opens the .raw portion of a Message and returns it as a string -func (m *FileMessage) ReadRaw() (raw *string, err error) { +func (m *Message) ReadRaw() (raw *string, err error) { reader, err := m.RawReader() if err != nil { return nil, err @@ -163,10 +163,10 @@ func (m *FileMessage) ReadRaw() (raw *string, err error) { // Append data to a newly opened Message, this will fail on a pre-existing Message and // after Close() is called. -func (m *FileMessage) Append(data []byte) error { +func (m *Message) Append(data []byte) error { // Prevent Appending to a pre-existing Message if !m.writable { - return datastore.ErrNotWritable + return storage.ErrNotWritable } // Open file for writing if we haven't yet if m.writer == nil { @@ -190,7 +190,7 @@ func (m *FileMessage) Append(data []byte) error { // Close this Message for writing - no more data may be Appended. Close() will also // trigger the creation of the .gob file. -func (m *FileMessage) Close() error { +func (m *Message) Close() error { // nil out the writer fields so they can't be used writer := m.writer writerFile := m.writerFile @@ -245,7 +245,7 @@ func (m *FileMessage) Close() error { // Delete this Message from disk by removing it from the index and deleting the // raw files. -func (m *FileMessage) Delete() error { +func (m *Message) Delete() error { messages := m.mailbox.messages for i, mm := range messages { if m == mm { diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 7d7dd35..5588cbb 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bufio" @@ -48,17 +48,17 @@ func countGenerator(c chan int) { } } -// FileDataStore implements DataStore aand is the root of the mail storage +// Store implements DataStore aand is the root of the mail storage // hiearchy. It provides access to Mailbox objects -type FileDataStore struct { - hashLock datastore.HashLock +type Store struct { + hashLock storage.HashLock path string mailPath string messageCap int } -// NewFileDataStore creates a new DataStore object using the specified path -func NewFileDataStore(cfg config.DataStoreConfig) datastore.DataStore { +// New creates a new DataStore object using the specified path +func New(cfg config.DataStoreConfig) storage.Store { path := cfg.Path if path == "" { log.Errorf("No value configured for datastore path") @@ -71,19 +71,19 @@ func NewFileDataStore(cfg config.DataStoreConfig) datastore.DataStore { log.Errorf("Error creating dir %q: %v", mailPath, err) } } - return &FileDataStore{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} + return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } -// DefaultFileDataStore creates a new DataStore object. It uses the inbucket.Config object to +// DefaultStore creates a new DataStore object. It uses the inbucket.Config object to // construct it's path. -func DefaultFileDataStore() datastore.DataStore { +func DefaultStore() storage.Store { cfg := config.GetDataStoreConfig() - return NewFileDataStore(cfg) + return New(cfg) } // MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox // does not exist, it will attempt to create it. -func (ds *FileDataStore) MailboxFor(emailAddress string) (datastore.Mailbox, error) { +func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -94,13 +94,13 @@ func (ds *FileDataStore) MailboxFor(emailAddress string) (datastore.Mailbox, err path := filepath.Join(ds.mailPath, s1, s2, dir) indexPath := filepath.Join(path, indexFileName) - return &FileMailbox{store: ds, name: name, dirName: dir, path: path, + return &Mailbox{store: ds, name: name, dirName: dir, path: path, indexPath: indexPath}, nil } // AllMailboxes returns a slice with all Mailboxes -func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { - mailboxes := make([]datastore.Mailbox, 0, 100) +func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { + mailboxes := make([]storage.Mailbox, 0, 100) infos1, err := ioutil.ReadDir(ds.mailPath) if err != nil { return nil, err @@ -127,7 +127,7 @@ func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { mbdir := inf3.Name() mbpath := filepath.Join(ds.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &FileMailbox{store: ds, dirName: mbdir, path: mbpath, + mb := &Mailbox{store: ds, dirName: mbdir, path: mbpath, indexPath: idx} mailboxes = append(mailboxes, mb) } @@ -141,7 +141,7 @@ func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { } // LockFor returns the RWMutex for this mailbox, or an error. -func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) { +func (ds *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -150,38 +150,38 @@ func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) { return ds.hashLock.Get(hash), nil } -// FileMailbox implements Mailbox, manages the mail for a specific user and +// Mailbox implements Mailbox, manages the mail for a specific user and // correlates to a particular directory on disk. -type FileMailbox struct { - store *FileDataStore +type Mailbox struct { + store *Store name string dirName string path string indexLoaded bool indexPath string - messages []*FileMessage + messages []*Message } // Name of the mailbox -func (mb *FileMailbox) Name() string { +func (mb *Mailbox) Name() string { return mb.name } // String renders the name and directory path of the mailbox -func (mb *FileMailbox) String() string { +func (mb *Mailbox) String() string { return mb.name + "[" + mb.dirName + "]" } // GetMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *FileMailbox) GetMessages() ([]datastore.Message, error) { +func (mb *Mailbox) GetMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]datastore.Message, len(mb.messages)) + messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -189,7 +189,7 @@ func (mb *FileMailbox) GetMessages() ([]datastore.Message, error) { } // GetMessage decodes a single message by Id and returns a Message object -func (mb *FileMailbox) GetMessage(id string) (datastore.Message, error) { +func (mb *Mailbox) GetMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err @@ -206,17 +206,17 @@ func (mb *FileMailbox) GetMessage(id string) (datastore.Message, error) { } } - return nil, datastore.ErrNotExist + return nil, storage.ErrNotExist } // Purge deletes all messages in this mailbox -func (mb *FileMailbox) Purge() error { +func (mb *Mailbox) Purge() error { mb.messages = mb.messages[:0] return mb.writeIndex() } // readIndex loads the mailbox index data from disk -func (mb *FileMailbox) readIndex() error { +func (mb *Mailbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] // Lock for reading @@ -242,7 +242,7 @@ func (mb *FileMailbox) readIndex() error { // Decode gob data dec := gob.NewDecoder(bufio.NewReader(file)) for { - msg := new(FileMessage) + msg := new(Message) if err = dec.Decode(msg); err != nil { if err == io.EOF { // It's OK to get an EOF here @@ -259,7 +259,7 @@ func (mb *FileMailbox) readIndex() error { } // writeIndex overwrites the index on disk with the current mailbox data -func (mb *FileMailbox) writeIndex() error { +func (mb *Mailbox) writeIndex() error { // Lock for writing indexMx.Lock() defer indexMx.Unlock() @@ -301,7 +301,7 @@ func (mb *FileMailbox) writeIndex() error { } // createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *FileMailbox) createDir() error { +func (mb *Mailbox) createDir() error { dirMx.Lock() defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { @@ -314,7 +314,7 @@ func (mb *FileMailbox) createDir() error { } // removeDir removes the mailbox, plus empty higher level directories -func (mb *FileMailbox) removeDir() error { +func (mb *Mailbox) removeDir() error { dirMx.Lock() defer dirMx.Unlock() // remove mailbox dir, including index file diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index b9fd41c..b02bdd1 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bytes" @@ -359,7 +359,7 @@ func TestFSMissing(t *testing.T) { // Delete a message file without removing it from index msg, err := mb.GetMessage(sentIds[1]) assert.Nil(t, err) - fmsg := msg.(*FileMessage) + fmsg := msg.(*Message) _ = os.Remove(fmsg.rawPath()) msg, err = mb.GetMessage(sentIds[1]) assert.Nil(t, err) @@ -508,7 +508,7 @@ func TestGetLatestMessage(t *testing.T) { } // setupDataStore creates a new FileDataStore in a temporary directory -func setupDataStore(cfg config.DataStoreConfig) (*FileDataStore, *bytes.Buffer) { +func setupDataStore(cfg config.DataStoreConfig) (*Store, *bytes.Buffer) { path, err := ioutil.TempDir("", "inbucket") if err != nil { panic(err) @@ -519,12 +519,12 @@ func setupDataStore(cfg config.DataStoreConfig) (*FileDataStore, *bytes.Buffer) log.SetOutput(buf) cfg.Path = path - return NewFileDataStore(cfg).(*FileDataStore), buf + return New(cfg).(*Store), buf } // deliverMessage creates and delivers a message to the specific mailbox, returning // the size of the generated message. -func deliverMessage(ds *FileDataStore, mbName string, subject string, +func deliverMessage(ds *Store, mbName string, subject string, date time.Time) (id string, size int64) { // Build fake SMTP message for delivery testMsg := make([]byte, 0, 300) @@ -544,7 +544,7 @@ func deliverMessage(ds *FileDataStore, mbName string, subject string, if err != nil { panic(err) } - fmsg := msg.(*FileMessage) + fmsg := msg.(*Message) fmsg.Fdate = date fmsg.Fid = id if err = msg.Append(testMsg); err != nil { @@ -557,7 +557,7 @@ func deliverMessage(ds *FileDataStore, mbName string, subject string, return id, int64(len(testMsg)) } -func teardownDataStore(ds *FileDataStore) { +func teardownDataStore(ds *Store) { if err := os.RemoveAll(ds.path); err != nil { panic(err) } diff --git a/pkg/storage/lock.go b/pkg/storage/lock.go index 5247702..8612e80 100644 --- a/pkg/storage/lock.go +++ b/pkg/storage/lock.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "strconv" diff --git a/pkg/storage/lock_test.go b/pkg/storage/lock_test.go index 203ebf1..ab87bba 100644 --- a/pkg/storage/lock_test.go +++ b/pkg/storage/lock_test.go @@ -1,4 +1,4 @@ -package datastore_test +package storage_test import ( "testing" @@ -7,7 +7,7 @@ import ( ) func TestHashLock(t *testing.T) { - hl := &datastore.HashLock{} + hl := &storage.HashLock{} // Invalid hashes testCases := []struct { diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index ef3154b..f067843 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "container/list" @@ -47,14 +47,14 @@ func init() { type RetentionScanner struct { globalShutdown chan bool // Closes when Inbucket needs to shut down retentionShutdown chan bool // Closed after the scanner has shut down - ds DataStore + ds Store retentionPeriod time.Duration retentionSleep time.Duration } // NewRetentionScanner launches a go-routine that scans for expired // messages, following the configured interval -func NewRetentionScanner(ds DataStore, shutdownChannel chan bool) *RetentionScanner { +func NewRetentionScanner(ds Store, shutdownChannel chan bool) *RetentionScanner { cfg := config.GetDataStoreConfig() rs := &RetentionScanner{ globalShutdown: shutdownChannel, diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index c357f7e..ae221e9 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "fmt" diff --git a/pkg/storage/datastore.go b/pkg/storage/storage.go similarity index 87% rename from pkg/storage/datastore.go rename to pkg/storage/storage.go index a9bcb57..a137f9b 100644 --- a/pkg/storage/datastore.go +++ b/pkg/storage/storage.go @@ -1,5 +1,5 @@ -// Package datastore contains implementation independent datastore logic -package datastore +// Package storage contains implementation independent datastore logic +package storage import ( "errors" @@ -19,8 +19,8 @@ var ( ErrNotWritable = errors.New("Message not writable") ) -// DataStore is an interface to get Mailboxes stored in Inbucket -type DataStore interface { +// Store is an interface to get Mailboxes stored in Inbucket +type Store interface { MailboxFor(emailAddress string) (Mailbox, error) AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index aa6c3de..fc8dbab 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "io" diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index bc2ef77..f9684b9 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -105,7 +105,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } msg, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -154,7 +154,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -191,7 +191,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -237,7 +237,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -290,7 +290,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } From 9c18f1fb30e69e077a7ca65f9e96ee118e828521 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 18:50:18 -0800 Subject: [PATCH 06/82] Large refactor for #69 - makefile: Don't refresh deps automatically, causes double build - storage: Move GetMessage, GetMessages (Mailbox), PurgeMessages to the Store API for #69 - storage: Remove Mailbox.Name method for #69 - test: Create new test package for #79 - test: Implement StoreStub, migrate some tests off MockDataStore for task #80 - rest & webui: update controllers to use new Store methods --- .travis.yml | 1 + Makefile | 4 +- pkg/rest/apiv1_controller.go | 35 ++---------- pkg/rest/apiv1_controller_test.go | 71 ++++-------------------- pkg/server/pop3/handler.go | 7 +-- pkg/server/smtp/handler.go | 7 ++- pkg/storage/file/fstore.go | 49 +++++++++++++---- pkg/storage/file/fstore_test.go | 91 +++++++------------------------ pkg/storage/storage.go | 6 +- pkg/storage/testing.go | 24 ++++++-- pkg/test/storage.go | 47 ++++++++++++++++ pkg/webui/mailbox_controller.go | 45 +++------------ 12 files changed, 160 insertions(+), 227 deletions(-) create mode 100644 pkg/test/storage.go diff --git a/.travis.yml b/.travis.yml index c820ee3..817ab3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ env: before_script: - go get github.com/golang/lint/golint + - make deps go: - "1.10" diff --git a/Makefile b/Makefile index c5104cc..821c674 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,9 @@ clean: deps: go get -t ./... -build: deps $(commands) +build: $(commands) -test: deps +test: go test -race ./... fmt: diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 010bfac..9d7d1c6 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,12 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - messages, err := mb.GetMessages() + messages, err := ctx.DataStore.GetMessages(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -59,12 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - msg, err := mb.GetMessage(id) + msg, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -121,13 +111,8 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } // Delete all messages - err = mb.Purge() + err = ctx.DataStore.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -144,12 +129,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -178,12 +158,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 9c4e789..a48fd38 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -2,14 +2,13 @@ package rest import ( "encoding/json" - "fmt" "io" "net/mail" "os" "testing" "time" - "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) const ( @@ -31,7 +30,7 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := &storage.MockDataStore{} + ds := test.NewStore() logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -45,10 +44,6 @@ func TestRestMailboxList(t *testing.T) { } // Test empty mailbox - emptybox := &storage.MockMailbox{} - ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessages").Return([]storage.Message{}, nil) - w, err = testRestGet(baseURL + "/mailbox/empty") expectCode = 200 if err != nil { @@ -58,30 +53,8 @@ func TestRestMailboxList(t *testing.T) { t.Errorf("Expected code %v, got %v", expectCode, w.Code) } - // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) - w, err = testRestGet(baseURL + "/mailbox/error") - expectCode = 500 - if err != nil { - t.Fatal(err) - } - if w.Code != expectCode { - t.Errorf("Expected code %v, got %v", expectCode, w.Code) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } - - // Test MailboxFor error - error2box := &storage.MockMailbox{} - ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessages").Return([]storage.Message{}, fmt.Errorf("Internal error 2")) - - w, err = testRestGet(baseURL + "/mailbox/error2") + // Test Mailbox error + w, err = testRestGet(baseURL + "/mailbox/messageserr") expectCode = 500 if err != nil { t.Fatal(err) @@ -107,11 +80,10 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - goodbox := &storage.MockMailbox{} - ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() msg2 := data2.MockMessage() - goodbox.On("GetMessages").Return([]storage.Message{msg1, msg2}, nil) + ds.AddMessage("good", msg1) + ds.AddMessage("good", msg2) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -130,7 +102,7 @@ func TestRestMailboxList(t *testing.T) { t.Errorf("Failed to decode JSON: %v", err) } if len(result) != 2 { - t.Errorf("Expected 2 results, got %v", len(result)) + t.Fatalf("Expected 2 results, got %v", len(result)) } if errors := data1.CompareToJSONHeaderMap(result[0]); len(errors) > 0 { t.Logf("%v", result[0]) @@ -155,7 +127,7 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := &storage.MockDataStore{} + ds := test.NewStore() logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -169,10 +141,6 @@ func TestRestMessage(t *testing.T) { } // Test requesting a message that does not exist - emptybox := &storage.MockMailbox{} - ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessage", "0001").Return(&storage.MockMessage{}, storage.ErrNotExist) - w, err = testRestGet(baseURL + "/mailbox/empty/0001") expectCode = 404 if err != nil { @@ -182,9 +150,8 @@ func TestRestMessage(t *testing.T) { t.Errorf("Expected code %v, got %v", expectCode, w.Code) } - // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) - w, err = testRestGet(baseURL + "/mailbox/error/0001") + // Test GetMessage error + w, err = testRestGet(baseURL + "/mailbox/messageerr/0001") expectCode = 500 if err != nil { t.Fatal(err) @@ -200,20 +167,6 @@ func TestRestMessage(t *testing.T) { _, _ = io.Copy(os.Stderr, logbuf) } - // Test GetMessage error - error2box := &storage.MockMailbox{} - ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessage", "0001").Return(&storage.MockMessage{}, fmt.Errorf("Internal error 2")) - - w, err = testRestGet(baseURL + "/mailbox/error2/0001") - expectCode = 500 - if err != nil { - t.Fatal(err) - } - if w.Code != expectCode { - t.Errorf("Expected code %v, got %v", expectCode, w.Code) - } - // Test JSON message headers data1 := &InputMessageData{ Mailbox: "good", @@ -228,10 +181,8 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - goodbox := &storage.MockMailbox{} - ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() - goodbox.On("GetMessage", "0001").Return(msg1, nil) + ds.AddMessage("good", msg1) // Check return code w, err = testRestGet(baseURL + "/mailbox/good/0001") diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 98cce65..4f6c4b6 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -513,12 +513,11 @@ func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { // Load the users mailbox func (ses *Session) loadMailbox() { - var err error - ses.messages, err = ses.mailbox.GetMessages() + m, err := ses.server.dataStore.GetMessages(ses.user) if err != nil { - ses.logError("Failed to load messages for %v", ses.user) + ses.logError("Failed to load messages for %v: %v", ses.user, err) } - + ses.messages = m ses.retainAll() } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index a678d3a..3b0660d 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -478,10 +478,15 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) ss.logError("Error while closing message for %v: %v", r.mailbox, err) return false } + name, err := stringutil.ParseMailboxName(r.localPart) + if err != nil { + // This parse already succeeded when MailboxFor was called, shouldn't fail here. + return false + } // Broadcast message information broadcast := msghub.Message{ - Mailbox: r.mailbox.Name(), + Mailbox: name, ID: msg.ID(), From: msg.From(), To: msg.To(), diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 5588cbb..0a03873 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -81,9 +81,36 @@ func DefaultStore() storage.Store { return New(cfg) } +// GetMessage returns the messages in the named mailbox, or an error. +func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).GetMessage(id) +} + +// GetMessages returns the messages in the named mailbox, or an error. +func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).GetMessages() +} + +// PurgeMessages deletes all messages in the named mailbox, or returns an error. +func (fs *Store) PurgeMessages(name string) error { + mb, err := fs.MailboxFor(name) + if err != nil { + return err + } + return mb.(*Mailbox).Purge() +} + // MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox // does not exist, it will attempt to create it. -func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { +func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -91,17 +118,17 @@ func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { dir := stringutil.HashMailboxName(name) s1 := dir[0:3] s2 := dir[0:6] - path := filepath.Join(ds.mailPath, s1, s2, dir) + path := filepath.Join(fs.mailPath, s1, s2, dir) indexPath := filepath.Join(path, indexFileName) - return &Mailbox{store: ds, name: name, dirName: dir, path: path, + return &Mailbox{store: fs, name: name, dirName: dir, path: path, indexPath: indexPath}, nil } // AllMailboxes returns a slice with all Mailboxes -func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { +func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { mailboxes := make([]storage.Mailbox, 0, 100) - infos1, err := ioutil.ReadDir(ds.mailPath) + infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return nil, err } @@ -109,7 +136,7 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf1 := range infos1 { if inf1.IsDir() { l1 := inf1.Name() - infos2, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1)) + infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) if err != nil { return nil, err } @@ -117,7 +144,7 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf2 := range infos2 { if inf2.IsDir() { l2 := inf2.Name() - infos3, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1, l2)) + infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) if err != nil { return nil, err } @@ -125,9 +152,9 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf3 := range infos3 { if inf3.IsDir() { mbdir := inf3.Name() - mbpath := filepath.Join(ds.mailPath, l1, l2, mbdir) + mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &Mailbox{store: ds, dirName: mbdir, path: mbpath, + mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} mailboxes = append(mailboxes, mb) } @@ -141,13 +168,13 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { } // LockFor returns the RWMutex for this mailbox, or an error. -func (ds *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { +func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err } hash := stringutil.HashMailboxName(name) - return ds.hashLock.Get(hash), nil + return fs.hashLock.Get(hash), nil } // Mailbox implements Mailbox, manages the mail for a specific user and diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index b02bdd1..26671c4 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -62,9 +62,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - mb, err := ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err := mb.GetMessage(id1) + msg, err := ds.GetMessage(mbName, id1) assert.Nil(t, err) err = msg.Delete() assert.Nil(t, err) @@ -76,7 +74,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err = mb.GetMessage(id2) + msg, err = ds.GetMessage(mbName, id2) assert.Nil(t, err) err = msg.Delete() assert.Nil(t, err) @@ -137,11 +135,7 @@ func TestFSDeliverMany(t *testing.T) { for i, subj := range subjects { // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -151,11 +145,7 @@ func TestFSDeliverMany(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -189,11 +179,7 @@ func TestFSDelete(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -205,11 +191,7 @@ func TestFSDelete(t *testing.T) { _ = msgs[3].Delete() // Confirm deletion - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -225,11 +207,7 @@ func TestFSDelete(t *testing.T) { // Try appending one more deliverMessage(ds, mbName, "foxtrot", time.Now()) - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -263,11 +241,7 @@ func TestFSPurge(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -275,15 +249,11 @@ func TestFSPurge(t *testing.T) { len(subjects), len(msgs)) // Purge mailbox - err = mb.Purge() + err = ds.PurgeMessages(mbName) assert.Nil(t, err) // Confirm deletion - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -315,12 +285,8 @@ func TestFSSize(t *testing.T) { sentSizes[i] = size } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } for i, id := range sentIds { - msg, err := mb.GetMessage(id) + msg, err := ds.GetMessage(mbName, id) assert.Nil(t, err) expect := sentSizes[i] @@ -351,17 +317,12 @@ func TestFSMissing(t *testing.T) { sentIds[i] = id } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - // Delete a message file without removing it from index - msg, err := mb.GetMessage(sentIds[1]) + msg, err := ds.GetMessage(mbName, sentIds[1]) assert.Nil(t, err) fmsg := msg.(*Message) _ = os.Remove(fmsg.rawPath()) - msg, err = mb.GetMessage(sentIds[1]) + msg, err = ds.GetMessage(mbName, sentIds[1]) assert.Nil(t, err) // Try to read parts of message @@ -392,11 +353,7 @@ func TestFSMessageCap(t *testing.T) { t.Logf("Delivered %q", subj) // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -437,11 +394,7 @@ func TestFSNoMessageCap(t *testing.T) { t.Logf("Delivered %q", subj) // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -467,9 +420,7 @@ func TestGetLatestMessage(t *testing.T) { mbName := "james" // Test empty mailbox - mb, err := ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err := mb.GetMessage("latest") + msg, err := ds.GetMessage(mbName, "latest") assert.Nil(t, msg) assert.Error(t, err) @@ -480,23 +431,19 @@ func TestGetLatestMessage(t *testing.T) { id2, _ := deliverMessage(ds, mbName, "test 2", time.Now()) // Test get the latest message - mb, err = ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err = mb.GetMessage("latest") + msg, err = ds.GetMessage(mbName, "latest") assert.Nil(t, err) assert.True(t, msg.ID() == id2, "Expected %q to be equal to %q", msg.ID(), id2) // Deliver test message 3 id3, _ := deliverMessage(ds, mbName, "test 3", time.Now()) - mb, err = ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err = mb.GetMessage("latest") + msg, err = ds.GetMessage(mbName, "latest") assert.Nil(t, err) assert.True(t, msg.ID() == id3, "Expected %q to be equal to %q", msg.ID(), id3) // Test wrong id - _, err = mb.GetMessage("wrongid") + _, err = ds.GetMessage(mbName, "wrongid") assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a137f9b..51620ed 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -21,6 +21,9 @@ var ( // Store is an interface to get Mailboxes stored in Inbucket type Store interface { + GetMessage(mailbox string, id string) (Message, error) + GetMessages(mailbox string) ([]Message, error) + PurgeMessages(mailbox string) error MailboxFor(emailAddress string) (Mailbox, error) AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp @@ -30,10 +33,7 @@ type Store interface { // Mailbox is an interface to get and manipulate messages in a DataStore type Mailbox interface { GetMessages() ([]Message, error) - GetMessage(id string) (Message, error) - Purge() error NewMessage() (Message, error) - Name() string String() string } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index fc8dbab..8c51b3f 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -15,6 +15,24 @@ type MockDataStore struct { mock.Mock } +// GetMessage mock function +func (m *MockDataStore) GetMessage(name, id string) (Message, error) { + args := m.Called(name, id) + return args.Get(0).(Message), args.Error(1) +} + +// GetMessages mock function +func (m *MockDataStore) GetMessages(name string) ([]Message, error) { + args := m.Called(name) + return args.Get(0).([]Message), args.Error(1) +} + +// PurgeMessages mock function +func (m *MockDataStore) PurgeMessages(name string) error { + args := m.Called(name) + return args.Error(0) +} + // MailboxFor mock function func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { args := m.Called(name) @@ -61,12 +79,6 @@ func (m *MockMailbox) NewMessage() (Message, error) { return args.Get(0).(Message), args.Error(1) } -// Name mock function -func (m *MockMailbox) Name() string { - args := m.Called() - return args.String(0) -} - // String mock function func (m *MockMailbox) String() string { args := m.Called() diff --git a/pkg/test/storage.go b/pkg/test/storage.go new file mode 100644 index 0000000..29c24ce --- /dev/null +++ b/pkg/test/storage.go @@ -0,0 +1,47 @@ +package test + +import ( + "errors" + + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// StoreStub stubs storage.Store for testing. +type StoreStub struct { + storage.Store + mailboxes map[string][]storage.Message +} + +// NewStore creates a new StoreStub. +func NewStore() *StoreStub { + return &StoreStub{ + mailboxes: make(map[string][]storage.Message), + } +} + +// AddMessage adds a message to the specified mailbox. +func (s *StoreStub) AddMessage(mailbox string, m storage.Message) { + msgs := s.mailboxes[mailbox] + s.mailboxes[mailbox] = append(msgs, m) +} + +// GetMessage gets a message by ID from the specified mailbox. +func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { + if mailbox == "messageerr" { + return nil, errors.New("internal error") + } + for _, m := range s.mailboxes[mailbox] { + if m.ID() == id { + return m, nil + } + } + return nil, storage.ErrNotExist +} + +// GetMessages gets all the messages for the specified mailbox. +func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { + if mailbox == "messageserr" { + return nil, errors.New("internal error") + } + return s.mailboxes[mailbox], nil +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index f9684b9..69b3a3f 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,12 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - messages, err := mb.GetMessages() + messages, err := ctx.DataStore.GetMessages(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -99,12 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - msg, err := mb.GetMessage(id) + msg, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -148,12 +138,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -172,8 +157,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er "ctx": ctx, "name": name, "message": message, - // TODO It is not really safe to render, need to sanitize, issue #5 - "body": template.HTML(mime.HTML), + "body": template.HTML(mime.HTML), }) } @@ -185,12 +169,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -231,12 +210,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -284,12 +258,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil From d9b5e40c8764b4b04264584cf61d23940632d7e7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 22:05:10 -0800 Subject: [PATCH 07/82] storage: More refactoring for #69 - retention: Start from pkg main instead of server/smtp - file: Remove DefaultStore() constructor - storage: AllMailboxes replaced with VisitMailboxes for #69 - test: Stub VisitMailboxes for #80 --- cmd/inbucket/main.go | 9 ++++-- pkg/server/smtp/listener.go | 34 +++++++++------------ pkg/storage/file/fstore.go | 30 +++++++++---------- pkg/storage/file/fstore_test.go | 21 +++++++++---- pkg/storage/retention.go | 42 ++++++++++++-------------- pkg/storage/retention_test.go | 52 +++++++++++++-------------------- pkg/storage/storage.go | 2 +- pkg/storage/testing.go | 12 ++++---- pkg/test/storage.go | 11 +++++++ 9 files changed, 106 insertions(+), 107 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 6e25e44..9f1d116 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -19,6 +19,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/server/pop3" "github.com/jhillyerd/inbucket/pkg/server/smtp" "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/storage/file" "github.com/jhillyerd/inbucket/pkg/webui" ) @@ -115,8 +116,11 @@ func main() { // Create message hub msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) - // Grab our datastore - ds := file.DefaultStore() + // Setup our datastore + dscfg := config.GetDataStoreConfig() + ds := file.New(dscfg) + retentionScanner := storage.NewRetentionScanner(dscfg, ds, shutdownChan) + retentionScanner.Start() // Start HTTP server web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) @@ -160,6 +164,7 @@ signalLoop: go timedExit() smtpServer.Drain() pop3Server.Drain() + retentionScanner.Join() removePIDFile() } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 4586174..83d5697 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -48,10 +48,9 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info - retentionScanner *storage.RetentionScanner // Deletes expired messages + dataStore storage.Store // Mailbox/message store + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info // State listener net.Listener // Incoming network connections @@ -86,18 +85,17 @@ func NewServer( ds storage.Store, msgHub *msghub.Hub) *Server { return &Server{ - host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), - domain: cfg.Domain, - domainNoStore: strings.ToLower(cfg.DomainNoStore), - maxRecips: cfg.MaxRecipients, - maxIdleSeconds: cfg.MaxIdleSeconds, - maxMessageBytes: cfg.MaxMessageBytes, - storeMessages: cfg.StoreMessages, - globalShutdown: globalShutdown, - dataStore: ds, - msgHub: msgHub, - retentionScanner: storage.NewRetentionScanner(ds, globalShutdown), - waitgroup: new(sync.WaitGroup), + host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), + domain: cfg.Domain, + domainNoStore: strings.ToLower(cfg.DomainNoStore), + maxRecips: cfg.MaxRecipients, + maxIdleSeconds: cfg.MaxIdleSeconds, + maxMessageBytes: cfg.MaxMessageBytes, + storeMessages: cfg.StoreMessages, + globalShutdown: globalShutdown, + dataStore: ds, + msgHub: msgHub, + waitgroup: new(sync.WaitGroup), } } @@ -124,9 +122,6 @@ func (s *Server) Start(ctx context.Context) { log.Infof("Messages sent to domain '%v' will be discarded", s.domainNoStore) } - // Start retention scanner - s.retentionScanner.Start() - // Listener go routine go s.serve(ctx) @@ -195,5 +190,4 @@ func (s *Server) Drain() { // Wait for sessions to close s.waitgroup.Wait() log.Tracef("SMTP connections have drained") - s.retentionScanner.Join() } diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 0a03873..ddf6d1c 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -74,13 +74,6 @@ func New(cfg config.DataStoreConfig) storage.Store { return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } -// DefaultStore creates a new DataStore object. It uses the inbucket.Config object to -// construct it's path. -func DefaultStore() storage.Store { - cfg := config.GetDataStoreConfig() - return New(cfg) -} - // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { mb, err := fs.MailboxFor(mailbox) @@ -125,12 +118,12 @@ func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { indexPath: indexPath}, nil } -// AllMailboxes returns a slice with all Mailboxes -func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { - mailboxes := make([]storage.Mailbox, 0, 100) +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { - return nil, err + return err } // Loop over level 1 directories for _, inf1 := range infos1 { @@ -138,7 +131,7 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { l1 := inf1.Name() infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) if err != nil { - return nil, err + return err } // Loop over level 2 directories for _, inf2 := range infos2 { @@ -146,7 +139,7 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { l2 := inf2.Name() infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) if err != nil { - return nil, err + return err } // Loop over mailboxes for _, inf3 := range infos3 { @@ -156,15 +149,20 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { idx := filepath.Join(mbpath, indexFileName) mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} - mailboxes = append(mailboxes, mb) + msgs, err := mb.GetMessages() + if err != nil { + return err + } + if !f(msgs) { + return nil + } } } } } } } - - return mailboxes, nil + return nil } // LockFor returns the RWMutex for this mailbox, or an error. diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 26671c4..8247de9 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/storage" "github.com/stretchr/testify/assert" ) @@ -97,12 +98,12 @@ func TestFSDirStructure(t *testing.T) { } } -// Test FileDataStore.AllMailboxes() -func TestFSAllMailboxes(t *testing.T) { +// TestFSVisitMailboxes tests VisitMailboxes +func TestFSVisitMailboxes(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) defer teardownDataStore(ds) - - for _, name := range []string{"abby", "bill", "christa", "donald", "evelyn"} { + boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} + for _, name := range boxes { // Create day old message date := time.Now().Add(-24 * time.Hour) deliverMessage(ds, name, "Old Message", date) @@ -112,9 +113,17 @@ func TestFSAllMailboxes(t *testing.T) { deliverMessage(ds, name, "New Message", date) } - mboxes, err := ds.AllMailboxes() + seen := 0 + err := ds.VisitMailboxes(func(messages []storage.Message) bool { + seen++ + count := len(messages) + if count != 2 { + t.Errorf("got: %v messages, want: 2", count) + } + return true + }) assert.Nil(t, err) - assert.Equal(t, len(mboxes), 5) + assert.Equal(t, 5, seen) if t.Failed() { // Wait for handler to finish logging diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index f067843..6269443 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -52,10 +52,12 @@ type RetentionScanner struct { retentionSleep time.Duration } -// NewRetentionScanner launches a go-routine that scans for expired -// messages, following the configured interval -func NewRetentionScanner(ds Store, shutdownChannel chan bool) *RetentionScanner { - cfg := config.GetDataStoreConfig() +// NewRetentionScanner configures a new RententionScanner. +func NewRetentionScanner( + cfg config.DataStoreConfig, + ds Store, + shutdownChannel chan bool, +) *RetentionScanner { rs := &RetentionScanner{ globalShutdown: shutdownChannel, retentionShutdown: make(chan bool), @@ -97,7 +99,7 @@ retentionLoop: } // Kickoff scan start = time.Now() - if err := rs.doScan(); err != nil { + if err := rs.DoScan(); err != nil { log.Errorf("Error during retention scan: %v", err) } // Check for global shutdown @@ -111,28 +113,17 @@ retentionLoop: close(rs.retentionShutdown) } -// doScan does a single pass of all mailboxes looking for messages that can be purged -func (rs *RetentionScanner) doScan() error { +// DoScan does a single pass of all mailboxes looking for messages that can be purged. +func (rs *RetentionScanner) DoScan() error { log.Tracef("Starting retention scan") cutoff := time.Now().Add(-1 * rs.retentionPeriod) - mboxes, err := rs.ds.AllMailboxes() - if err != nil { - return err - } retained := 0 - // Loop over all mailboxes - for _, mb := range mboxes { - messages, err := mb.GetMessages() - if err != nil { - return err - } - // Loop over all messages in mailbox + // Loop over all mailboxes. + err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { log.Tracef("Purging expired message %v", msg.ID()) - err = msg.Delete() - if err != nil { - // Log but don't abort + if err := msg.Delete(); err != nil { log.Errorf("Failed to purge message %v: %v", msg.ID(), err) } else { expRetentionDeletesTotal.Add(1) @@ -141,14 +132,17 @@ func (rs *RetentionScanner) doScan() error { retained++ } } - // Sleep after completing a mailbox select { case <-rs.globalShutdown: log.Tracef("Retention scan aborted due to shutdown") - return nil + return false case <-time.After(rs.retentionSleep): // Reduce disk thrashing } + return true + }) + if err != nil { + return err } // Update metrics setRetentionScanCompleted(time.Now()) @@ -156,7 +150,7 @@ func (rs *RetentionScanner) doScan() error { return nil } -// Join does not retun until the retention scanner has shut down +// Join does not return until the retention scanner has shut down. func (rs *RetentionScanner) Join() { if rs.retentionShutdown != nil { <-rs.retentionShutdown diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index ae221e9..bd862cf 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -1,19 +1,17 @@ -package storage +package storage_test import ( "fmt" "testing" "time" + + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) func TestDoRetentionScan(t *testing.T) { - // Create mock objects - mds := &MockDataStore{} - - mb1 := &MockMailbox{} - mb2 := &MockMailbox{} - mb3 := &MockMailbox{} - + ds := test.NewStore() // Mockup some different aged messages (num is in hours) new1 := mockMessage(0) new2 := mockMessage(1) @@ -21,36 +19,26 @@ func TestDoRetentionScan(t *testing.T) { old1 := mockMessage(4) old2 := mockMessage(12) old3 := mockMessage(24) - - // First it should ask for all mailboxes - mds.On("AllMailboxes").Return([]Mailbox{mb1, mb2, mb3}, nil) - - // Then for all messages on each box - mb1.On("GetMessages").Return([]Message{new1, old1, old2}, nil) - mb2.On("GetMessages").Return([]Message{old3, new2}, nil) - mb3.On("GetMessages").Return([]Message{new3}, nil) - + ds.AddMessage("mb1", new1) + ds.AddMessage("mb1", old1) + ds.AddMessage("mb1", old2) + ds.AddMessage("mb2", old3) + ds.AddMessage("mb2", new2) + ds.AddMessage("mb3", new3) // Test 4 hour retention - rs := &RetentionScanner{ - ds: mds, - retentionPeriod: 4*time.Hour - time.Minute, - retentionSleep: 0, + cfg := config.DataStoreConfig{ + RetentionMinutes: 239, + RetentionSleep: 0, } - if err := rs.doScan(); err != nil { + shutdownChan := make(chan bool) + rs := storage.NewRetentionScanner(cfg, ds, shutdownChan) + if err := rs.DoScan(); err != nil { t.Error(err) } - - // Check our assertions - mds.AssertExpectations(t) - mb1.AssertExpectations(t) - mb2.AssertExpectations(t) - mb3.AssertExpectations(t) - // Delete should not have been called on new messages new1.AssertNotCalled(t, "Delete") new2.AssertNotCalled(t, "Delete") new3.AssertNotCalled(t, "Delete") - // Delete should have been called once on old messages old1.AssertNumberOfCalls(t, "Delete", 1) old2.AssertNumberOfCalls(t, "Delete", 1) @@ -58,8 +46,8 @@ func TestDoRetentionScan(t *testing.T) { } // Make a MockMessage of a specific age -func mockMessage(ageHours int) *MockMessage { - msg := &MockMessage{} +func mockMessage(ageHours int) *storage.MockMessage { + msg := &storage.MockMessage{} msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) msg.On("Delete").Return(nil) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 51620ed..b18bf33 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -24,8 +24,8 @@ type Store interface { GetMessage(mailbox string, id string) (Message, error) GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error + VisitMailboxes(f func([]Message) (cont bool)) error MailboxFor(emailAddress string) (Mailbox, error) - AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 8c51b3f..64d7bf0 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -39,17 +39,17 @@ func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { return args.Get(0).(Mailbox), args.Error(1) } -// AllMailboxes mock function -func (m *MockDataStore) AllMailboxes() ([]Mailbox, error) { - args := m.Called() - return args.Get(0).([]Mailbox), args.Error(1) -} - // LockFor mock function returns a new RWMutex, never errors. func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil } +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { + return nil +} + // MockMailbox is a shared mock for unit testing type MockMailbox struct { mock.Mock diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 29c24ce..3f0fcbd 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -45,3 +45,14 @@ func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { } return s.mailboxes[mailbox], nil } + +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { + for _, v := range s.mailboxes { + if !f(v) { + return nil + } + } + return nil +} From 137466f89b4cc6f0cac1d46ec9b80946f5117ca2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 10:48:50 -0700 Subject: [PATCH 08/82] storage: Move NewMessage() into Store interface for #69 --- pkg/server/smtp/handler.go | 2 +- pkg/server/smtp/handler_test.go | 4 ++-- pkg/storage/file/fstore.go | 9 +++++++++ pkg/storage/file/fstore_test.go | 6 +----- pkg/storage/storage.go | 3 ++- pkg/storage/testing.go | 6 ++++++ pkg/test/storage.go | 5 +++++ 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 3b0660d..228718a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -451,7 +451,7 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { - msg, err := r.mailbox.NewMessage() + msg, err := ss.server.dataStore.NewMessage(r.localPart) if err != nil { ss.logError("Failed to create message for %q: %s", r.localPart, err) return false diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index b53b651..b8603a5 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -148,7 +148,7 @@ func TestMailState(t *testing.T) { mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) - mb1.On("NewMessage").Return(msg1, nil) + mds.On("NewMessage", "u1").Return(msg1, nil) mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") @@ -263,7 +263,7 @@ func TestDataState(t *testing.T) { mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) - mb1.On("NewMessage").Return(msg1, nil) + mds.On("NewMessage", "u1").Return(msg1, nil) mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index ddf6d1c..b3a2e5e 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -175,6 +175,15 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { return fs.hashLock.Get(hash), nil } +// NewMessage is temproary until #69 MessageData refactor +func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).NewMessage() +} + // Mailbox implements Mailbox, manages the mail for a specific user and // correlates to a particular directory on disk. type Mailbox struct { diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 8247de9..a7bb039 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -490,13 +490,9 @@ func deliverMessage(ds *Store, mbName string, subject string, testMsg = append(testMsg, []byte("\r\n")...) testMsg = append(testMsg, []byte("Test Body\r\n")...) - mb, err := ds.MailboxFor(mbName) - if err != nil { - panic(err) - } // Create message object id = generateID(date) - msg, err := mb.NewMessage() + msg, err := ds.NewMessage(mbName) if err != nil { panic(err) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index b18bf33..54adb58 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,12 +28,13 @@ type Store interface { MailboxFor(emailAddress string) (Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) + // NewMessage is temproary until #69 MessageData refactor + NewMessage(mailbox string) (Message, error) } // Mailbox is an interface to get and manipulate messages in a DataStore type Mailbox interface { GetMessages() ([]Message, error) - NewMessage() (Message, error) String() string } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 64d7bf0..8757230 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -44,6 +44,12 @@ func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil } +// NewMessage temporary for #69 +func (m *MockDataStore) NewMessage(mailbox string) (Message, error) { + args := m.Called(mailbox) + return args.Get(0).(Message), args.Error(1) +} + // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 3f0fcbd..fab78ca 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -56,3 +56,8 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error } return nil } + +// NewMessage is temproary until #69 MessageData refactor +func (s *StoreStub) NewMessage(mailbox string) (storage.Message, error) { + return nil, nil +} From 12ad0cb3f0abec74c191e3ab62293792b1062cae Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 11:54:35 -0700 Subject: [PATCH 09/82] storage: Eliminate storage.Mailbox interface for #69 storage/file Mailbox has been renamed mbox, and is now just an implementation detail. --- pkg/server/pop3/handler.go | 17 ------ pkg/server/smtp/handler.go | 15 ++---- pkg/server/smtp/handler_test.go | 6 --- pkg/storage/file/fmessage.go | 8 ++- pkg/storage/file/fstore.go | 94 ++++++++++++++------------------- pkg/storage/storage.go | 7 --- pkg/storage/testing.go | 41 -------------- 7 files changed, 45 insertions(+), 143 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 4f6c4b6..5e3c665 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -65,7 +65,6 @@ type Session struct { state State // Current session state reader *bufio.Reader // Buffered reader for our net conn user string // Mailbox name - mailbox storage.Mailbox // Mailbox instance messages []storage.Message // Slice of messages in mailbox retain []bool // Messages to retain upon UPDATE (true=retain) msgCount int // Number of undeleted messages @@ -195,14 +194,6 @@ func (ses *Session) authorizationHandler(cmd string, args []string) { if ses.user == "" { ses.ooSeq(cmd) } else { - var err error - ses.mailbox, err = ses.server.dataStore.MailboxFor(ses.user) - if err != nil { - ses.logError("Failed to open mailbox for %v", ses.user) - ses.send(fmt.Sprintf("-ERR Failed to open mailbox for %v", ses.user)) - ses.enterState(QUIT) - return - } ses.loadMailbox() ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) ses.enterState(TRANSACTION) @@ -214,14 +205,6 @@ func (ses *Session) authorizationHandler(cmd string, args []string) { return } ses.user = args[0] - var err error - ses.mailbox, err = ses.server.dataStore.MailboxFor(ses.user) - if err != nil { - ses.logError("Failed to open mailbox for %v", ses.user) - ses.send(fmt.Sprintf("-ERR Failed to open mailbox for %v", ses.user)) - ses.enterState(QUIT) - return - } ses.loadMailbox() ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) ses.enterState(TRANSACTION) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 228718a..50bf9ef 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -14,7 +14,6 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -73,7 +72,6 @@ var commands = map[string]bool{ // recipientDetails for message delivery type recipientDetails struct { address, localPart, domainPart string - mailbox storage.Mailbox } // Session holds the state of an SMTP session @@ -365,14 +363,7 @@ func (ss *Session) dataHandler() { } if strings.ToLower(domain) != ss.server.domainNoStore { // Not our "no store" domain, so store the message - mb, err := ss.server.dataStore.MailboxFor(local) - if err != nil { - ss.logError("Failed to open mailbox for %q: %s", local, err) - ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", local)) - ss.reset() - return - } - recipients = append(recipients, recipientDetails{recip, local, domain, mb}) + recipients = append(recipients, recipientDetails{recip, local, domain}) } else { log.Tracef("Not storing message for %q", recip) } @@ -469,13 +460,13 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) // Append lines from msgBuf for _, line := range msgBuf { if err := msg.Append(line); err != nil { - ss.logError("Failed to append to mailbox %v: %v", r.mailbox, err) + ss.logError("Failed to append to mailbox %v: %v", r.localPart, err) // Should really cleanup the crap on filesystem return false } } if err := msg.Close(); err != nil { - ss.logError("Error while closing message for %v: %v", r.mailbox, err) + ss.logError("Error while closing message for %v: %v", r.localPart, err) return false } name, err := stringutil.ParseMailboxName(r.localPart) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index b8603a5..29098ac 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -145,11 +145,8 @@ func TestReadyState(t *testing.T) { func TestMailState(t *testing.T) { // Setup mock objects mds := &storage.MockDataStore{} - mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} - mds.On("MailboxFor", "u1").Return(mb1, nil) mds.On("NewMessage", "u1").Return(msg1, nil) - mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") msg1.On("To").Return(make([]string, 0)) @@ -260,11 +257,8 @@ func TestMailState(t *testing.T) { func TestDataState(t *testing.T) { // Setup mock objects mds := &storage.MockDataStore{} - mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} - mds.On("MailboxFor", "u1").Return(mb1, nil) mds.On("NewMessage", "u1").Return(msg1, nil) - mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") msg1.On("To").Return(make([]string, 0)) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 4e496f8..b325678 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -18,7 +18,7 @@ import ( // Message implements Message and contains a little bit of data about a // particular email message, and methods to retrieve the rest of it from disk. type Message struct { - mailbox *Mailbox + mailbox *mbox // Stored in GOB Fid string Fdate time.Time @@ -32,16 +32,15 @@ type Message struct { writer *bufio.Writer } -// NewMessage creates a new FileMessage object and sets the Date and Id fields. +// newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *Mailbox) NewMessage() (storage.Message, error) { +func (mb *mbox) newMessage() (storage.Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - // Delete old messages over messageCap if mb.store.messageCap > 0 { for len(mb.messages) >= mb.store.messageCap { @@ -51,7 +50,6 @@ func (mb *Mailbox) NewMessage() (storage.Message, error) { } } } - date := time.Now() id := generateID(date) return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index b3a2e5e..fb09ad0 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -76,46 +76,29 @@ func New(cfg config.DataStoreConfig) storage.Store { // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).GetMessage(id) + return mb.getMessage(id) } // GetMessages returns the messages in the named mailbox, or an error. func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).GetMessages() + return mb.getMessages() } // PurgeMessages deletes all messages in the named mailbox, or returns an error. -func (fs *Store) PurgeMessages(name string) error { - mb, err := fs.MailboxFor(name) +func (fs *Store) PurgeMessages(mailbox string) error { + mb, err := fs.mbox(mailbox) if err != nil { return err } - return mb.(*Mailbox).Purge() -} - -// MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox -// does not exist, it will attempt to create it. -func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { - name, err := stringutil.ParseMailboxName(emailAddress) - if err != nil { - return nil, err - } - dir := stringutil.HashMailboxName(name) - s1 := dir[0:3] - s2 := dir[0:6] - path := filepath.Join(fs.mailPath, s1, s2, dir) - indexPath := filepath.Join(path, indexFileName) - - return &Mailbox{store: fs, name: name, dirName: dir, path: path, - indexPath: indexPath}, nil + return mb.purge() } // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it @@ -147,9 +130,9 @@ func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { mbdir := inf3.Name() mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, + mb := &mbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} - msgs, err := mb.GetMessages() + msgs, err := mb.getMessages() if err != nil { return err } @@ -177,16 +160,31 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { // NewMessage is temproary until #69 MessageData refactor func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).NewMessage() + return mb.newMessage() } -// Mailbox implements Mailbox, manages the mail for a specific user and -// correlates to a particular directory on disk. -type Mailbox struct { +// mbox returns the named mailbox. +func (fs *Store) mbox(mailbox string) (*mbox, error) { + name, err := stringutil.ParseMailboxName(mailbox) + if err != nil { + return nil, err + } + dir := stringutil.HashMailboxName(name) + s1 := dir[0:3] + s2 := dir[0:6] + path := filepath.Join(fs.mailPath, s1, s2, dir) + indexPath := filepath.Join(path, indexFileName) + + return &mbox{store: fs, name: name, dirName: dir, path: path, + indexPath: indexPath}, nil +} + +// mbox manages the mail for a specific user and correlates to a particular directory on disk. +type mbox struct { store *Store name string dirName string @@ -196,25 +194,14 @@ type Mailbox struct { messages []*Message } -// Name of the mailbox -func (mb *Mailbox) Name() string { - return mb.name -} - -// String renders the name and directory path of the mailbox -func (mb *Mailbox) String() string { - return mb.name + "[" + mb.dirName + "]" -} - -// GetMessages scans the mailbox directory for .gob files and decodes them into +// getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *Mailbox) GetMessages() ([]storage.Message, error) { +func (mb *mbox) getMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m @@ -222,35 +209,32 @@ func (mb *Mailbox) GetMessages() ([]storage.Message, error) { return messages, nil } -// GetMessage decodes a single message by Id and returns a Message object -func (mb *Mailbox) GetMessage(id string) (storage.Message, error) { +// getMessage decodes a single message by ID and returns a Message object. +func (mb *mbox) getMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - if id == "latest" && len(mb.messages) != 0 { return mb.messages[len(mb.messages)-1], nil } - for _, m := range mb.messages { if m.Fid == id { return m, nil } } - return nil, storage.ErrNotExist } -// Purge deletes all messages in this mailbox -func (mb *Mailbox) Purge() error { +// purge deletes all messages in this mailbox. +func (mb *mbox) purge() error { mb.messages = mb.messages[:0] return mb.writeIndex() } // readIndex loads the mailbox index data from disk -func (mb *Mailbox) readIndex() error { +func (mb *mbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] // Lock for reading @@ -293,7 +277,7 @@ func (mb *Mailbox) readIndex() error { } // writeIndex overwrites the index on disk with the current mailbox data -func (mb *Mailbox) writeIndex() error { +func (mb *mbox) writeIndex() error { // Lock for writing indexMx.Lock() defer indexMx.Unlock() @@ -335,7 +319,7 @@ func (mb *Mailbox) writeIndex() error { } // createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *Mailbox) createDir() error { +func (mb *mbox) createDir() error { dirMx.Lock() defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { @@ -348,7 +332,7 @@ func (mb *Mailbox) createDir() error { } // removeDir removes the mailbox, plus empty higher level directories -func (mb *Mailbox) removeDir() error { +func (mb *mbox) removeDir() error { dirMx.Lock() defer dirMx.Unlock() // remove mailbox dir, including index file diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 54adb58..83cf635 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -25,19 +25,12 @@ type Store interface { GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error VisitMailboxes(f func([]Message) (cont bool)) error - MailboxFor(emailAddress string) (Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) // NewMessage is temproary until #69 MessageData refactor NewMessage(mailbox string) (Message, error) } -// Mailbox is an interface to get and manipulate messages in a DataStore -type Mailbox interface { - GetMessages() ([]Message, error) - String() string -} - // Message is an interface for a single message in a Mailbox type Message interface { ID() string diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 8757230..6b2604d 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -33,12 +33,6 @@ func (m *MockDataStore) PurgeMessages(name string) error { return args.Error(0) } -// MailboxFor mock function -func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { - args := m.Called(name) - return args.Get(0).(Mailbox), args.Error(1) -} - // LockFor mock function returns a new RWMutex, never errors. func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil @@ -56,41 +50,6 @@ func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { return nil } -// MockMailbox is a shared mock for unit testing -type MockMailbox struct { - mock.Mock -} - -// GetMessages mock function -func (m *MockMailbox) GetMessages() ([]Message, error) { - args := m.Called() - return args.Get(0).([]Message), args.Error(1) -} - -// GetMessage mock function -func (m *MockMailbox) GetMessage(id string) (Message, error) { - args := m.Called(id) - return args.Get(0).(Message), args.Error(1) -} - -// Purge mock function -func (m *MockMailbox) Purge() error { - args := m.Called() - return args.Error(0) -} - -// NewMessage mock function -func (m *MockMailbox) NewMessage() (Message, error) { - args := m.Called() - return args.Get(0).(Message), args.Error(1) -} - -// String mock function -func (m *MockMailbox) String() string { - args := m.Called() - return args.String(0) -} - // MockMessage is a shared mock for unit testing type MockMessage struct { mock.Mock From 487e491d6f0c32bf493f2b931ea56bebdf51b99f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 15:01:40 -0700 Subject: [PATCH 10/82] storage: Message refactoring for #69 - Message interface renamed to StoreMessage - Message.Delete becomes Store.RemoveMessage - Added deleted message tracking to Store stub for #80 --- pkg/rest/apiv1_controller.go | 8 +--- pkg/server/pop3/handler.go | 28 ++++++------- pkg/server/smtp/handler_test.go | 13 +++--- pkg/storage/file/fmessage.go | 35 ++++------------ pkg/storage/file/fstore.go | 74 ++++++++++++++++++++++++++------- pkg/storage/file/fstore_test.go | 20 +++++---- pkg/storage/retention.go | 6 +-- pkg/storage/retention_test.go | 22 +++++++--- pkg/storage/storage.go | 21 +++++----- pkg/storage/testing.go | 32 ++++++++------ pkg/test/storage.go | 49 ++++++++++++++++++---- 11 files changed, 190 insertions(+), 118 deletions(-) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 9d7d1c6..6424648 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -158,18 +158,14 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + err = ctx.DataStore.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) - } - err = message.Delete() - if err != nil { - return fmt.Errorf("Delete(%q) failed: %v", id, err) + return fmt.Errorf("RemoveMessage(%q) failed: %v", id, err) } return web.RenderJSON(w, "OK") diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 5e3c665..c022619 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,17 +57,17 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - messages []storage.Message // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + messages []storage.StoreMessage // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -415,7 +415,7 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg storage.Message) { +func (ses *Session) sendMessage(msg storage.StoreMessage) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -448,7 +448,7 @@ func (ses *Session) sendMessage(msg storage.Message) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { +func (ses *Session) sendMessageTop(msg storage.StoreMessage, lineCount int) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -522,7 +522,7 @@ func (ses *Session) processDeletes() { for i, msg := range ses.messages { if !ses.retain[i] { ses.logTrace("Deleting %v", msg) - if err := msg.Delete(); err != nil { + if err := ses.server.dataStore.RemoveMessage(ses.user, msg.ID()); err != nil { ses.logWarn("Error deleting %v: %v", msg, err) } } diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 29098ac..d516bf3 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) type scriptStep struct { @@ -25,10 +26,8 @@ type scriptStep struct { // Test commands in GREET state func TestGreetState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - - server, logbuf, teardown := setupSMTPServer(mds) + ds := test.NewStore() + server, logbuf, teardown := setupSMTPServer(ds) defer teardown() // Test out some mangled HELOs @@ -82,10 +81,8 @@ func TestGreetState(t *testing.T) { // Test commands in READY state func TestReadyState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - - server, logbuf, teardown := setupSMTPServer(mds) + ds := test.NewStore() + server, logbuf, teardown := setupSMTPServer(ds) defer teardown() // Test out some mangled READY commands diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index b325678..26c1612 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -34,7 +34,7 @@ type Message struct { // newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *mbox) newMessage() (storage.Message, error) { +func (mb *mbox) newMessage() (storage.StoreMessage, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -45,7 +45,7 @@ func (mb *mbox) newMessage() (storage.Message, error) { if mb.store.messageCap > 0 { for len(mb.messages) >= mb.store.messageCap { log.Infof("Mailbox %q over configured message cap", mb.name) - if err := mb.messages[0].Delete(); err != nil { + if err := mb.removeMessage(mb.messages[0].ID()); err != nil { log.Errorf("Error deleting message: %s", err) } } @@ -55,6 +55,11 @@ func (mb *mbox) newMessage() (storage.Message, error) { return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil } +// Mailbox returns the name of the mailbox this message resides in. +func (m *Message) Mailbox() string { + return m.mailbox.name +} + // ID gets the ID of the Message func (m *Message) ID() string { return m.Fid @@ -240,29 +245,3 @@ func (m *Message) Close() error { m.mailbox.messages = append(m.mailbox.messages, m) return m.mailbox.writeIndex() } - -// Delete this Message from disk by removing it from the index and deleting the -// raw files. -func (m *Message) Delete() error { - messages := m.mailbox.messages - for i, mm := range messages { - if m == mm { - // Slice around message we are deleting - m.mailbox.messages = append(messages[:i], messages[i+1:]...) - break - } - } - if err := m.mailbox.writeIndex(); err != nil { - return err - } - - if len(m.mailbox.messages) == 0 { - // This was the last message, thus writeIndex() has removed the entire - // directory; we don't need to delete the raw file. - return nil - } - - // There are still messages in the index - log.Tracef("Deleting %v", m.rawPath()) - return os.Remove(m.rawPath()) -} diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index fb09ad0..3e5f9bb 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -75,7 +75,7 @@ func New(cfg config.DataStoreConfig) storage.Store { } // GetMessage returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { +func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { } // GetMessages returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { +func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -92,6 +92,15 @@ func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { return mb.getMessages() } +// RemoveMessage deletes a message by ID from the specified mailbox. +func (fs *Store) RemoveMessage(mailbox, id string) error { + mb, err := fs.mbox(mailbox) + if err != nil { + return err + } + return mb.removeMessage(id) +} + // PurgeMessages deletes all messages in the named mailbox, or returns an error. func (fs *Store) PurgeMessages(mailbox string) error { mb, err := fs.mbox(mailbox) @@ -103,7 +112,7 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { +func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return err @@ -159,7 +168,7 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { } // NewMessage is temproary until #69 MessageData refactor -func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { +func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -196,13 +205,13 @@ type mbox struct { // getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.Message, error) { +func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.Message, len(mb.messages)) + messages := make([]storage.StoreMessage, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -210,7 +219,7 @@ func (mb *mbox) getMessages() ([]storage.Message, error) { } // getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.Message, error) { +func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err @@ -227,6 +236,38 @@ func (mb *mbox) getMessage(id string) (storage.Message, error) { return nil, storage.ErrNotExist } +// removeMessage deletes the message off disk and removes it from the index. +func (mb *mbox) removeMessage(id string) error { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return err + } + } + var msg *Message + for i, m := range mb.messages { + if id == m.ID() { + msg = m + // Slice around message we are deleting + mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) + break + } + } + if msg == nil { + return storage.ErrNotExist + } + if err := mb.writeIndex(); err != nil { + return err + } + if len(mb.messages) == 0 { + // This was the last message, thus writeIndex() has removed the entire + // directory; we don't need to delete the raw file. + return nil + } + // There are still messages in the index + log.Tracef("Deleting %v", msg.rawPath()) + return os.Remove(msg.rawPath()) +} + // purge deletes all messages in this mailbox. func (mb *mbox) purge() error { mb.messages = mb.messages[:0] @@ -256,14 +297,18 @@ func (mb *mbox) readIndex() error { log.Errorf("Failed to close %q: %v", mb.indexPath, err) } }() - // Decode gob data dec := gob.NewDecoder(bufio.NewReader(file)) + name := "" + if err = dec.Decode(&name); err != nil { + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + mb.name = name for { - msg := new(Message) + // Load messages until EOF + msg := &Message{} if err = dec.Decode(msg); err != nil { if err == io.EOF { - // It's OK to get an EOF here break } return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) @@ -271,7 +316,6 @@ func (mb *mbox) readIndex() error { msg.mailbox = mb mb.messages = append(mb.messages, msg) } - mb.indexLoaded = true return nil } @@ -294,9 +338,12 @@ func (mb *mbox) writeIndex() error { writer := bufio.NewWriter(file) // Write each message and then flush enc := gob.NewEncoder(writer) + if err = enc.Encode(mb.name); err != nil { + _ = file.Close() + return err + } for _, m := range mb.messages { - err = enc.Encode(m) - if err != nil { + if err = enc.Encode(m); err != nil { _ = file.Close() return err } @@ -314,7 +361,6 @@ func (mb *mbox) writeIndex() error { log.Tracef("Removing mailbox %v", mb.path) return mb.removeDir() } - return nil } diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index a7bb039..8db7253 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -63,9 +63,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err := ds.GetMessage(mbName, id1) - assert.Nil(t, err) - err = msg.Delete() + err := ds.RemoveMessage(mbName, id1) assert.Nil(t, err) // Message should be removed @@ -75,9 +73,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err = ds.GetMessage(mbName, id2) - assert.Nil(t, err) - err = msg.Delete() + err = ds.RemoveMessage(mbName, id2) assert.Nil(t, err) // Message should be removed @@ -114,7 +110,7 @@ func TestFSVisitMailboxes(t *testing.T) { } seen := 0 - err := ds.VisitMailboxes(func(messages []storage.Message) bool { + err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { seen++ count := len(messages) if count != 2 { @@ -196,8 +192,14 @@ func TestFSDelete(t *testing.T) { len(subjects), len(msgs)) // Delete a couple messages - _ = msgs[1].Delete() - _ = msgs[3].Delete() + err = ds.RemoveMessage(mbName, msgs[1].ID()) + if err != nil { + t.Fatal(err) + } + err = ds.RemoveMessage(mbName, msgs[3].ID()) + if err != nil { + t.Fatal(err) + } // Confirm deletion msgs, err = ds.GetMessages(mbName) diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 6269443..2da706e 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -119,11 +119,11 @@ func (rs *RetentionScanner) DoScan() error { cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 // Loop over all mailboxes. - err := rs.ds.VisitMailboxes(func(messages []Message) bool { + err := rs.ds.VisitMailboxes(func(messages []StoreMessage) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { - log.Tracef("Purging expired message %v", msg.ID()) - if err := msg.Delete(); err != nil { + log.Tracef("Purging expired message %v/%v", msg.Mailbox(), msg.ID()) + if err := rs.ds.RemoveMessage(msg.Mailbox(), msg.ID()); err != nil { log.Errorf("Failed to purge message %v: %v", msg.ID(), err) } else { expRetentionDeletesTotal.Add(1) diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index bd862cf..f6ed828 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -20,11 +20,17 @@ func TestDoRetentionScan(t *testing.T) { old2 := mockMessage(12) old3 := mockMessage(24) ds.AddMessage("mb1", new1) + new1.On("Mailbox").Return("mb1") ds.AddMessage("mb1", old1) + old1.On("Mailbox").Return("mb1") ds.AddMessage("mb1", old2) + old2.On("Mailbox").Return("mb1") ds.AddMessage("mb2", old3) + old3.On("Mailbox").Return("mb2") ds.AddMessage("mb2", new2) + new2.On("Mailbox").Return("mb2") ds.AddMessage("mb3", new3) + new3.On("Mailbox").Return("mb3") // Test 4 hour retention cfg := config.DataStoreConfig{ RetentionMinutes: 239, @@ -36,13 +42,17 @@ func TestDoRetentionScan(t *testing.T) { t.Error(err) } // Delete should not have been called on new messages - new1.AssertNotCalled(t, "Delete") - new2.AssertNotCalled(t, "Delete") - new3.AssertNotCalled(t, "Delete") + for _, m := range []storage.StoreMessage{new1, new2, new3} { + if ds.MessageDeleted(m) { + t.Errorf("Expected %v to be present, was deleted", m.ID()) + } + } // Delete should have been called once on old messages - old1.AssertNumberOfCalls(t, "Delete", 1) - old2.AssertNumberOfCalls(t, "Delete", 1) - old3.AssertNumberOfCalls(t, "Delete", 1) + for _, m := range []storage.StoreMessage{old1, old2, old3} { + if !ds.MessageDeleted(m) { + t.Errorf("Expected %v to be deleted, was present", m.ID()) + } + } } // Make a MockMessage of a specific age diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 83cf635..425a792 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -12,27 +12,29 @@ import ( ) var ( - // ErrNotExist indicates the requested message does not exist - ErrNotExist = errors.New("Message does not exist") + // ErrNotExist indicates the requested message does not exist. + ErrNotExist = errors.New("message does not exist") // ErrNotWritable indicates the message is closed; no longer writable ErrNotWritable = errors.New("Message not writable") ) -// Store is an interface to get Mailboxes stored in Inbucket +// Store is the interface Inbucket uses to interact with storage implementations. type Store interface { - GetMessage(mailbox string, id string) (Message, error) - GetMessages(mailbox string) ([]Message, error) + GetMessage(mailbox, id string) (StoreMessage, error) + GetMessages(mailbox string) ([]StoreMessage, error) PurgeMessages(mailbox string) error - VisitMailboxes(f func([]Message) (cont bool)) error + RemoveMessage(mailbox, id string) error + VisitMailboxes(f func([]StoreMessage) (cont bool)) error // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) // NewMessage is temproary until #69 MessageData refactor - NewMessage(mailbox string) (Message, error) + NewMessage(mailbox string) (StoreMessage, error) } -// Message is an interface for a single message in a Mailbox -type Message interface { +// StoreMessage represents a message to be stored, or returned from a storage implementation. +type StoreMessage interface { + Mailbox() string ID() string From() string To() []string @@ -44,7 +46,6 @@ type Message interface { ReadRaw() (raw *string, err error) Append(data []byte) error Close() error - Delete() error String() string Size() int64 } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 6b2604d..16c40b3 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -16,15 +16,21 @@ type MockDataStore struct { } // GetMessage mock function -func (m *MockDataStore) GetMessage(name, id string) (Message, error) { +func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { args := m.Called(name, id) - return args.Get(0).(Message), args.Error(1) + return args.Get(0).(StoreMessage), args.Error(1) } // GetMessages mock function -func (m *MockDataStore) GetMessages(name string) ([]Message, error) { +func (m *MockDataStore) GetMessages(name string) ([]StoreMessage, error) { args := m.Called(name) - return args.Get(0).([]Message), args.Error(1) + return args.Get(0).([]StoreMessage), args.Error(1) +} + +// RemoveMessage mock function +func (m *MockDataStore) RemoveMessage(name, id string) error { + args := m.Called(name, id) + return args.Error(0) } // PurgeMessages mock function @@ -39,14 +45,14 @@ func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { } // NewMessage temporary for #69 -func (m *MockDataStore) NewMessage(mailbox string) (Message, error) { +func (m *MockDataStore) NewMessage(mailbox string) (StoreMessage, error) { args := m.Called(mailbox) - return args.Get(0).(Message), args.Error(1) + return args.Get(0).(StoreMessage), args.Error(1) } // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { +func (m *MockDataStore) VisitMailboxes(f func([]StoreMessage) (cont bool)) error { return nil } @@ -55,6 +61,12 @@ type MockMessage struct { mock.Mock } +// Mailbox mock function +func (m *MockMessage) Mailbox() string { + args := m.Called() + return args.String(0) +} + // ID mock function func (m *MockMessage) ID() string { args := m.Called() @@ -127,12 +139,6 @@ func (m *MockMessage) Close() error { return args.Error(0) } -// Delete mock function -func (m *MockMessage) Delete() error { - args := m.Called() - return args.Error(0) -} - // String mock function func (m *MockMessage) String() string { args := m.Called() diff --git a/pkg/test/storage.go b/pkg/test/storage.go index fab78ca..92195ff 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -2,6 +2,7 @@ package test import ( "errors" + "sync" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -9,24 +10,26 @@ import ( // StoreStub stubs storage.Store for testing. type StoreStub struct { storage.Store - mailboxes map[string][]storage.Message + mailboxes map[string][]storage.StoreMessage + deleted map[storage.StoreMessage]struct{} } // NewStore creates a new StoreStub. func NewStore() *StoreStub { return &StoreStub{ - mailboxes: make(map[string][]storage.Message), + mailboxes: make(map[string][]storage.StoreMessage), + deleted: make(map[storage.StoreMessage]struct{}), } } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(mailbox string, m storage.Message) { +func (s *StoreStub) AddMessage(mailbox string, m storage.StoreMessage) { msgs := s.mailboxes[mailbox] s.mailboxes[mailbox] = append(msgs, m) } // GetMessage gets a message by ID from the specified mailbox. -func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { +func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) { if mailbox == "messageerr" { return nil, errors.New("internal error") } @@ -39,16 +42,36 @@ func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { } // GetMessages gets all the messages for the specified mailbox. -func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { +func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) { if mailbox == "messageserr" { return nil, errors.New("internal error") } return s.mailboxes[mailbox], nil } +// RemoveMessage deletes a message by ID from the specified mailbox. +func (s *StoreStub) RemoveMessage(mailbox, id string) error { + mb, ok := s.mailboxes[mailbox] + if ok { + var msg storage.StoreMessage + for i, m := range mb { + if m.ID() == id { + msg = m + s.mailboxes[mailbox] = append(mb[:i], mb[i+1:]...) + break + } + } + if msg != nil { + s.deleted[msg] = struct{}{} + return nil + } + } + return storage.ErrNotExist +} + // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { +func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { for _, v := range s.mailboxes { if !f(v) { return nil @@ -58,6 +81,18 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error } // NewMessage is temproary until #69 MessageData refactor -func (s *StoreStub) NewMessage(mailbox string) (storage.Message, error) { +func (s *StoreStub) NewMessage(mailbox string) (storage.StoreMessage, error) { return nil, nil } + +// LockFor mock function returns a new RWMutex, never errors. +// TODO(#69) remove +func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { + return &sync.RWMutex{}, nil +} + +// MessageDeleted returns true if the specified message was deleted +func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { + _, ok := s.deleted[m] + return ok +} From 3bc66d278893447f1b941ca69517670c94088d80 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 16:56:09 -0700 Subject: [PATCH 11/82] storage: Store addresses as mail.Address for #69 --- pkg/rest/apiv1_controller.go | 8 ++++---- pkg/rest/apiv1_controller_test.go | 10 +++++----- pkg/rest/testutils_test.go | 9 +++++++-- pkg/server/smtp/handler.go | 4 ++-- pkg/server/smtp/handler_test.go | 9 +++++---- pkg/storage/file/fmessage.go | 18 ++++++++---------- pkg/storage/storage.go | 4 ++-- pkg/storage/testing.go | 8 ++++---- pkg/stringutil/utils.go | 12 ++++++++++++ 9 files changed, 49 insertions(+), 33 deletions(-) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 6424648..bc03bfd 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -36,8 +36,8 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( jmessages[i] = &model.JSONMessageHeaderV1{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), @@ -90,8 +90,8 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( &model.JSONMessageV1{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index a48fd38..8053a48 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -67,16 +67,16 @@ func TestRestMailboxList(t *testing.T) { data1 := &InputMessageData{ Mailbox: "good", ID: "0001", - From: "from1", - To: []string{"to1"}, + From: "", + To: []string{""}, Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), } data2 := &InputMessageData{ Mailbox: "good", ID: "0002", - From: "from2", - To: []string{"to1"}, + From: "", + To: []string{""}, Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } @@ -171,7 +171,7 @@ func TestRestMessage(t *testing.T) { data1 := &InputMessageData{ Mailbox: "good", ID: "0001", - From: "from1", + From: "", Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), Header: mail.Header{ diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index ef294df..4b7d669 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -26,10 +26,15 @@ type InputMessageData struct { } func (d *InputMessageData) MockMessage() *storage.MockMessage { + from, _ := mail.ParseAddress(d.From) + to := make([]*mail.Address, len(d.To)) + for i, a := range d.To { + to[i], _ = mail.ParseAddress(a) + } msg := &storage.MockMessage{} msg.On("ID").Return(d.ID) - msg.On("From").Return(d.From) - msg.On("To").Return(d.To) + msg.On("From").Return(from) + msg.On("To").Return(to) msg.On("Subject").Return(d.Subject) msg.On("Date").Return(d.Date) msg.On("Size").Return(d.Size) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 50bf9ef..b1a6eb3 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -479,8 +479,8 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) broadcast := msghub.Message{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index d516bf3..ca9ff2e 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -8,6 +8,7 @@ import ( "log" "net" + "net/mail" "net/textproto" "os" "testing" @@ -145,8 +146,8 @@ func TestMailState(t *testing.T) { msg1 := &storage.MockMessage{} mds.On("NewMessage", "u1").Return(msg1, nil) msg1.On("ID").Return("") - msg1.On("From").Return("") - msg1.On("To").Return(make([]string, 0)) + msg1.On("From").Return(&mail.Address{}) + msg1.On("To").Return(make([]*mail.Address, 0)) msg1.On("Date").Return(time.Time{}) msg1.On("Subject").Return("") msg1.On("Size").Return(0) @@ -257,8 +258,8 @@ func TestDataState(t *testing.T) { msg1 := &storage.MockMessage{} mds.On("NewMessage", "u1").Return(msg1, nil) msg1.On("ID").Return("") - msg1.On("From").Return("") - msg1.On("To").Return(make([]string, 0)) + msg1.On("From").Return(&mail.Address{}) + msg1.On("To").Return(make([]*mail.Address, 0)) msg1.On("Date").Return(time.Time{}) msg1.On("Subject").Return("") msg1.On("Size").Return(0) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 26c1612..3f8b0d7 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -22,8 +22,8 @@ type Message struct { // Stored in GOB Fid string Fdate time.Time - Ffrom string - Fto []string + Ffrom *mail.Address + Fto []*mail.Address Fsubject string Fsize int64 // These are for creating new messages only @@ -71,12 +71,12 @@ func (m *Message) Date() time.Time { } // From returns the value of the Message From header -func (m *Message) From() string { +func (m *Message) From() *mail.Address { return m.Ffrom } // To returns the value of the Message To header -func (m *Message) To() []string { +func (m *Message) To() []*mail.Address { return m.Fto } @@ -220,19 +220,17 @@ func (m *Message) Close() error { // Only public fields are stored in gob, hence starting with capital F // Parse From address if address, err := mail.ParseAddress(body.GetHeader("From")); err == nil { - m.Ffrom = address.String() + m.Ffrom = address } else { - m.Ffrom = body.GetHeader("From") + m.Ffrom = &mail.Address{Address: body.GetHeader("From")} } m.Fsubject = body.GetHeader("Subject") // Turn the To header into a slice if addresses, err := body.AddressList("To"); err == nil { - for _, a := range addresses { - m.Fto = append(m.Fto, a.String()) - } + m.Fto = addresses } else { - m.Fto = []string{body.GetHeader("To")} + m.Fto = []*mail.Address{{Address: body.GetHeader("To")}} } // Refresh the index before adding our message diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 425a792..c3d2019 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -36,8 +36,8 @@ type Store interface { type StoreMessage interface { Mailbox() string ID() string - From() string - To() []string + From() *mail.Address + To() []*mail.Address Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 16c40b3..9defa55 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -74,15 +74,15 @@ func (m *MockMessage) ID() string { } // From mock function -func (m *MockMessage) From() string { +func (m *MockMessage) From() *mail.Address { args := m.Called() - return args.String(0) + return args.Get(0).(*mail.Address) } // To mock function -func (m *MockMessage) To() []string { +func (m *MockMessage) To() []*mail.Address { args := m.Called() - return args.Get(0).([]string) + return args.Get(0).([]*mail.Address) } // Date mock function diff --git a/pkg/stringutil/utils.go b/pkg/stringutil/utils.go index 450ae1f..193e552 100644 --- a/pkg/stringutil/utils.go +++ b/pkg/stringutil/utils.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" "fmt" "io" + "net/mail" "strings" ) @@ -224,3 +225,14 @@ LOOP: return buf.String(), domain, nil } + +// StringAddressList converts a list of addresses to a list of strings +func StringAddressList(addrs []*mail.Address) []string { + s := make([]string, len(addrs)) + for i, a := range addrs { + if a != nil { + s[i] = a.String() + } + } + return s +} From 10bc07a18e5cba41c4bc3523c1bd5fda467f6fd7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 22:25:21 -0700 Subject: [PATCH 12/82] message: Implement service layer, stubs for #81 I've made some effort to wire the manager into the controllers, but tests are currently failing. --- cmd/inbucket/main.go | 4 +- pkg/message/manager.go | 86 +++++++++++++++++++++++++++++++ pkg/message/message.go | 26 ++++++++++ pkg/rest/apiv1_controller.go | 60 +++++++++------------ pkg/rest/apiv1_controller_test.go | 75 ++++++++++++++++++++++++--- pkg/rest/model/apiv1_model.go | 3 +- pkg/rest/testutils_test.go | 5 +- pkg/server/web/context.go | 3 ++ pkg/server/web/server.go | 4 ++ pkg/storage/file/fmessage.go | 16 ------ pkg/storage/file/fstore_test.go | 2 - pkg/storage/storage.go | 1 - pkg/test/manager.go | 53 +++++++++++++++++++ pkg/webui/mailbox_controller.go | 64 ++++++++--------------- 14 files changed, 291 insertions(+), 111 deletions(-) create mode 100644 pkg/message/manager.go create mode 100644 pkg/message/message.go create mode 100644 pkg/test/manager.go diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 9f1d116..ff01d3f 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -14,6 +14,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/rest" "github.com/jhillyerd/inbucket/pkg/server/pop3" @@ -123,7 +124,8 @@ func main() { retentionScanner.Start() // Start HTTP server - web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) + mm := &message.StoreManager{Store: ds} + web.Initialize(config.GetWebConfig(), shutdownChan, mm, ds, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) diff --git a/pkg/message/manager.go b/pkg/message/manager.go new file mode 100644 index 0000000..5782273 --- /dev/null +++ b/pkg/message/manager.go @@ -0,0 +1,86 @@ +package message + +import ( + "io" + + "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// Manager is the interface controllers use to interact with messages. +type Manager interface { + GetMetadata(mailbox string) ([]*Metadata, error) + GetMessage(mailbox, id string) (*Message, error) + PurgeMessages(mailbox string) error + RemoveMessage(mailbox, id string) error + SourceReader(mailbox, id string) (io.ReadCloser, error) +} + +// StoreManager is a message Manager backed by the storage.Store. +type StoreManager struct { + Store storage.Store +} + +// GetMetadata returns a slice of metadata for the specified mailbox. +func (s *StoreManager) GetMetadata(mailbox string) ([]*Metadata, error) { + messages, err := s.Store.GetMessages(mailbox) + if err != nil { + return nil, err + } + metas := make([]*Metadata, len(messages)) + for i, sm := range messages { + metas[i] = makeMetadata(sm) + } + return metas, nil +} + +// GetMessage returns the specified message. +func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { + sm, err := s.Store.GetMessage(mailbox, id) + if err != nil { + return nil, err + } + r, err := sm.RawReader() + if err != nil { + return nil, err + } + env, err := enmime.ReadEnvelope(r) + if err != nil { + return nil, err + } + _ = r.Close() + header := makeMetadata(sm) + return &Message{Metadata: *header, Envelope: env}, nil +} + +// PurgeMessages removes all messages from the specified mailbox. +func (s *StoreManager) PurgeMessages(mailbox string) error { + return s.Store.PurgeMessages(mailbox) +} + +// RemoveMessage deletes the specified message. +func (s *StoreManager) RemoveMessage(mailbox, id string) error { + return s.Store.RemoveMessage(mailbox, id) +} + +// SourceReader allows the stored message source to be read. +func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { + sm, err := s.Store.GetMessage(mailbox, id) + if err != nil { + return nil, err + } + return sm.RawReader() +} + +// makeMetadata populates Metadata from a StoreMessage. +func makeMetadata(m storage.StoreMessage) *Metadata { + return &Metadata{ + Mailbox: m.Mailbox(), + ID: m.ID(), + From: m.From(), + To: m.To(), + Date: m.Date(), + Subject: m.Subject(), + Size: m.Size(), + } +} diff --git a/pkg/message/message.go b/pkg/message/message.go new file mode 100644 index 0000000..fd5f25b --- /dev/null +++ b/pkg/message/message.go @@ -0,0 +1,26 @@ +// Package message contains message handling logic. +package message + +import ( + "net/mail" + "time" + + "github.com/jhillyerd/enmime" +) + +// Metadata holds information about a message, but not the content. +type Metadata struct { + Mailbox string + ID string + From *mail.Address + To []*mail.Address + Date time.Time + Subject string + Size int64 +} + +// Message holds both the metadata and content of a message. +type Message struct { + Metadata + Envelope *enmime.Envelope +} diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index bc03bfd..135669e 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,7 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - messages, err := ctx.DataStore.GetMessages(name) + messages, err := ctx.MsgSvc.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -35,12 +35,12 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( for i, msg := range messages { jmessages[i] = &model.JSONMessageHeaderV1{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), + ID: msg.ID, + From: msg.From.String(), + To: stringutil.StringAddressList(msg.To), + Subject: msg.Subject, + Date: msg.Date, + Size: msg.Size, } } return web.RenderJSON(w, jmessages) @@ -54,7 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - msg, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -63,14 +63,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - header, err := msg.ReadHeader() - if err != nil { - return fmt.Errorf("ReadHeader(%q) failed: %v", id, err) - } - mime, err := msg.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope attachments := make([]*model.JSONMessageAttachmentV1, len(mime.Attachments)) for i, att := range mime.Attachments { @@ -89,13 +82,13 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return web.RenderJSON(w, &model.JSONMessageV1{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), - Header: header.Header, + ID: msg.ID, + From: msg.From.String(), + To: stringutil.StringAddressList(msg.To), + Subject: msg.Subject, + Date: msg.Date, + Size: msg.Size, + Header: mime.Root.Header, Body: &model.JSONMessageBodyV1{ Text: mime.Text, HTML: mime.HTML, @@ -112,7 +105,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } // Delete all messages - err = ctx.DataStore.PurgeMessages(name) + err = ctx.MsgSvc.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -129,25 +122,20 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + + r, err := ctx.MsgSvc.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) + return fmt.Errorf("SourceReader(%q) failed: %v", id, err) } - raw, err := message.ReadRaw() - if err != nil { - return fmt.Errorf("ReadRaw(%q) failed: %v", id, err) - } - + // Output message source w.Header().Set("Content-Type", "text/plain") - if _, err := io.WriteString(w, *raw); err != nil { - return err - } - return nil + _, err = io.Copy(w, r) + return err } // MailboxDeleteV1 removes a particular message from a mailbox @@ -158,7 +146,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - err = ctx.DataStore.RemoveMessage(name, id) + err = ctx.MsgSvc.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 8053a48..48d2841 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -4,10 +4,14 @@ import ( "encoding/json" "io" "net/mail" + "net/textproto" "os" + "strings" "testing" "time" + "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -31,7 +35,8 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup ds := test.NewStore() - logbuf := setupWebServer(ds) + mm := test.NewManager() + logbuf := setupWebServer(mm, ds) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar") @@ -80,10 +85,24 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - msg1 := data1.MockMessage() - msg2 := data2.MockMessage() - ds.AddMessage("good", msg1) - ds.AddMessage("good", msg2) + meta1 := message.Metadata{ + Mailbox: "good", + ID: "0001", + From: &mail.Address{Name: "", Address: "from1@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 1", + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + } + meta2 := message.Metadata{ + Mailbox: "good", + ID: "0002", + From: &mail.Address{Name: "", Address: "from2@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 2", + Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), + } + mm.AddMessage("good", &message.Message{Metadata: meta1}) + mm.AddMessage("good", &message.Message{Metadata: meta2}) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -96,6 +115,25 @@ func TestRestMailboxList(t *testing.T) { } // Check JSON + got := w.Body.String() + testStrings := []string{ + `{"mailbox":"good","id":"0001","from":"\u003cfrom1@host\u003e",` + + `"to":["\u003cto1@host\u003e"],"subject":"subject 1",` + + `"date":"2012-02-01T10:11:12.000000253-00:13","size":0}`, + `{"mailbox":"good","id":"0002","from":"\u003cfrom2@host\u003e",` + + `"to":["\u003cto1@host\u003e"],"subject":"subject 2",` + + `"date":"2012-07-01T10:11:12.000000253-00:11","size":0}`, + } + for _, ts := range testStrings { + t.Run(ts, func(t *testing.T) { + if !strings.Contains(got, ts) { + t.Errorf("got:\n%s\nwant to contain:\n%s", got, ts) + } + }) + } + + // Check JSON + // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result []interface{} if err := dec.Decode(&result); err != nil { @@ -128,7 +166,8 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup ds := test.NewStore() - logbuf := setupWebServer(ds) + mm := test.NewManager() + logbuf := setupWebServer(mm, ds) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar/0001") @@ -168,6 +207,26 @@ func TestRestMessage(t *testing.T) { } // Test JSON message headers + msg1 := &message.Message{ + Metadata: message.Metadata{ + Mailbox: "good", + ID: "0001", + From: &mail.Address{Name: "", Address: "from1@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 1", + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + }, + Envelope: &enmime.Envelope{ + Text: "This is some text", + HTML: "This is some HTML", + Root: &enmime.Part{ + Header: textproto.MIMEHeader{ + "To": []string{"fred@fish.com", "keyword@nsa.gov"}, + "From": []string{"noreply@inbucket.org"}, + }, + }, + }, + } data1 := &InputMessageData{ Mailbox: "good", ID: "0001", @@ -181,8 +240,7 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - msg1 := data1.MockMessage() - ds.AddMessage("good", msg1) + mm.AddMessage("good", msg1) // Check return code w, err = testRestGet(baseURL + "/mailbox/good/0001") @@ -195,6 +253,7 @@ func TestRestMessage(t *testing.T) { } // Check JSON + // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result map[string]interface{} if err := dec.Decode(&result); err != nil { diff --git a/pkg/rest/model/apiv1_model.go b/pkg/rest/model/apiv1_model.go index 1e4f7f0..7e1e083 100644 --- a/pkg/rest/model/apiv1_model.go +++ b/pkg/rest/model/apiv1_model.go @@ -1,7 +1,6 @@ package model import ( - "net/mail" "time" ) @@ -26,7 +25,7 @@ type JSONMessageV1 struct { Date time.Time `json:"date"` Size int64 `json:"size"` Body *JSONMessageBodyV1 `json:"body"` - Header mail.Header `json:"header"` + Header map[string][]string `json:"header"` Attachments []*JSONMessageAttachmentV1 `json:"attachments"` } diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 4b7d669..4bdcae5 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -11,6 +11,7 @@ import ( "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -193,7 +194,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(ds storage.Store) *bytes.Buffer { +func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) @@ -205,7 +206,7 @@ func setupWebServer(ds storage.Store) *bytes.Buffer { PublicDir: "../themes/bootstrap/public", } shutdownChan := make(chan bool) - web.Initialize(cfg, shutdownChan, ds, &msghub.Hub{}) + web.Initialize(cfg, shutdownChan, mm, ds, &msghub.Hub{}) SetupRoutes(web.Router) return buf diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index 422fdb5..af5f6b2 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/sessions" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -17,6 +18,7 @@ type Context struct { Session *sessions.Session DataStore storage.Store MsgHub *msghub.Hub + MsgSvc message.Manager WebConfig config.WebConfig IsJSON bool } @@ -61,6 +63,7 @@ func NewContext(req *http.Request) (*Context, error) { Session: sess, DataStore: DataStore, MsgHub: msgHub, + MsgSvc: msgSvc, WebConfig: webConfig, IsJSON: headerMatch(req, "Accept", "application/json"), } diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index 5c82430..c76a2f7 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -14,6 +14,7 @@ import ( "github.com/gorilla/sessions" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -27,6 +28,7 @@ var ( // msgHub holds a reference to the message pub/sub system msgHub *msghub.Hub + msgSvc message.Manager // Router is shared between httpd, webui and rest packages. It sends // incoming requests to the correct handler function @@ -51,6 +53,7 @@ func init() { func Initialize( cfg config.WebConfig, shutdownChan chan bool, + mm message.Manager, ds storage.Store, mh *msghub.Hub) { @@ -60,6 +63,7 @@ func Initialize( // NewContext() will use this DataStore for the web handlers DataStore = ds msgHub = mh + msgSvc = mm // Content Paths log.Infof("HTTP templates mapped to %q", cfg.TemplateDir) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 3f8b0d7..e9df65d 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -99,22 +99,6 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object -func (m *Message) ReadHeader() (msg *mail.Message, err error) { - file, err := os.Open(m.rawPath()) - if err != nil { - return nil, err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - reader := bufio.NewReader(file) - return mail.ReadMessage(reader) -} - // ReadBody opens the .raw portion of a Message and returns a MIMEBody object func (m *Message) ReadBody() (body *enmime.Envelope, err error) { file, err := os.Open(m.rawPath()) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 8db7253..eb5add1 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -337,8 +337,6 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.ReadHeader() - assert.Error(t, err) _, err = msg.ReadBody() assert.Error(t, err) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index c3d2019..52e50cc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -41,7 +41,6 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - ReadHeader() (msg *mail.Message, err error) ReadBody() (body *enmime.Envelope, err error) ReadRaw() (raw *string, err error) Append(data []byte) error diff --git a/pkg/test/manager.go b/pkg/test/manager.go new file mode 100644 index 0000000..47c87a8 --- /dev/null +++ b/pkg/test/manager.go @@ -0,0 +1,53 @@ +package test + +import ( + "errors" + + "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// ManagerStub is a test stub for message.Manager +type ManagerStub struct { + message.Manager + mailboxes map[string][]*message.Message +} + +// NewManager creates a new ManagerStub. +func NewManager() *ManagerStub { + return &ManagerStub{ + mailboxes: make(map[string][]*message.Message), + } +} + +// AddMessage adds a message to the specified mailbox. +func (m *ManagerStub) AddMessage(mailbox string, msg *message.Message) { + messages := m.mailboxes[mailbox] + m.mailboxes[mailbox] = append(messages, msg) +} + +// GetMessage gets a message by ID from the specified mailbox. +func (m *ManagerStub) GetMessage(mailbox, id string) (*message.Message, error) { + if mailbox == "messageerr" { + return nil, errors.New("internal error") + } + for _, msg := range m.mailboxes[mailbox] { + if msg.ID == id { + return msg, nil + } + } + return nil, storage.ErrNotExist +} + +// GetMetadata gets all the metadata for the specified mailbox. +func (m *ManagerStub) GetMetadata(mailbox string) ([]*message.Metadata, error) { + if mailbox == "messageserr" { + return nil, errors.New("internal error") + } + messages := m.mailboxes[mailbox] + metas := make([]*message.Metadata, len(messages)) + for i, msg := range messages { + metas[i] = &msg.Metadata + } + return metas, nil +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 69b3a3f..ed092e7 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,7 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - messages, err := ctx.DataStore.GetMessages(name) + messages, err := ctx.MsgSvc.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -94,7 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -103,10 +103,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime, err := msg.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope body := template.HTML(web.TextToHTML(mime.Text)) htmlAvailable := mime.HTML != "" var htmlBody template.HTML @@ -138,25 +135,22 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime, err := message.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope // Render partial template w.Header().Set("Content-Type", "text/html; charset=UTF-8") return web.RenderPartial("mailbox/_html.html", w, map[string]interface{}{ "ctx": ctx, "name": name, - "message": message, + "message": msg, "body": template.HTML(mime.HTML), }) } @@ -169,25 +163,19 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + r, err := ctx.MsgSvc.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) - } - raw, err := message.ReadRaw() - if err != nil { - return fmt.Errorf("ReadRaw(%q) failed: %v", id, err) + return fmt.Errorf("SourceReader(%q) failed: %v", id, err) } // Output message source w.Header().Set("Content-Type", "text/plain") - if _, err := io.WriteString(w, *raw); err != nil { - return err - } - return nil + _, err = io.Copy(w, r) + return err } // MailboxDownloadAttach sends the attachment to the client; disposition: @@ -210,19 +198,16 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body, err := message.ReadBody() - if err != nil { - return err - } + body := msg.Envelope if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) @@ -233,10 +218,8 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // Output attachment w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", "attachment") - if _, err := io.Copy(w, part); err != nil { - return err - } - return nil + _, err = io.Copy(w, part) + return err } // MailboxViewAttach sends the attachment to the client for online viewing @@ -258,19 +241,16 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body, err := message.ReadBody() - if err != nil { - return err - } + body := msg.Envelope if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) @@ -280,8 +260,6 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex part := body.Attachments[num] // Output attachment w.Header().Set("Content-Type", part.ContentType) - if _, err := io.Copy(w, part); err != nil { - return err - } - return nil + _, err = io.Copy(w, part) + return err } From 219862797e0eed58b82cc88f92b4e6944fdd10ab Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 12 Mar 2018 20:49:06 -0700 Subject: [PATCH 13/82] web: remove DataStore from Context and controllers for #81 --- cmd/inbucket/main.go | 2 +- pkg/rest/apiv1_controller.go | 10 +++++----- pkg/rest/apiv1_controller_test.go | 6 ++---- pkg/rest/testutils_test.go | 4 ++-- pkg/server/web/context.go | 7 ++----- pkg/server/web/server.go | 12 +++--------- pkg/webui/mailbox_controller.go | 12 ++++++------ 7 files changed, 21 insertions(+), 32 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index ff01d3f..7020676 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -125,7 +125,7 @@ func main() { // Start HTTP server mm := &message.StoreManager{Store: ds} - web.Initialize(config.GetWebConfig(), shutdownChan, mm, ds, msgHub) + web.Initialize(config.GetWebConfig(), shutdownChan, mm, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 135669e..7b40201 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,7 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - messages, err := ctx.MsgSvc.GetMetadata(name) + messages, err := ctx.Manager.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -54,7 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -105,7 +105,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } // Delete all messages - err = ctx.MsgSvc.PurgeMessages(name) + err = ctx.Manager.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -123,7 +123,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } - r, err := ctx.MsgSvc.SourceReader(name, id) + r, err := ctx.Manager.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -146,7 +146,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - err = ctx.MsgSvc.RemoveMessage(name, id) + err = ctx.Manager.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 48d2841..b31cfcb 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -34,9 +34,8 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := test.NewStore() mm := test.NewManager() - logbuf := setupWebServer(mm, ds) + logbuf := setupWebServer(mm) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar") @@ -165,9 +164,8 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := test.NewStore() mm := test.NewManager() - logbuf := setupWebServer(mm, ds) + logbuf := setupWebServer(mm) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar/0001") diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 4bdcae5..cfddede 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -194,7 +194,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { +func setupWebServer(mm message.Manager) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) @@ -206,7 +206,7 @@ func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { PublicDir: "../themes/bootstrap/public", } shutdownChan := make(chan bool) - web.Initialize(cfg, shutdownChan, mm, ds, &msghub.Hub{}) + web.Initialize(cfg, shutdownChan, mm, &msghub.Hub{}) SetupRoutes(web.Router) return buf diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index af5f6b2..9ec2ab8 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -9,16 +9,14 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Context is passed into every request handler function type Context struct { Vars map[string]string Session *sessions.Session - DataStore storage.Store MsgHub *msghub.Hub - MsgSvc message.Manager + Manager message.Manager WebConfig config.WebConfig IsJSON bool } @@ -61,9 +59,8 @@ func NewContext(req *http.Request) (*Context, error) { ctx := &Context{ Vars: vars, Session: sess, - DataStore: DataStore, MsgHub: msgHub, - MsgSvc: msgSvc, + Manager: manager, WebConfig: webConfig, IsJSON: headerMatch(req, "Accept", "application/json"), } diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index c76a2f7..b3e8610 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -16,19 +16,15 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Handler is a function type that handles an HTTP request in Inbucket type Handler func(http.ResponseWriter, *http.Request, *Context) error var ( - // DataStore is where all the mailboxes and messages live - DataStore storage.Store - // msgHub holds a reference to the message pub/sub system - msgHub *msghub.Hub - msgSvc message.Manager + msgHub *msghub.Hub + manager message.Manager // Router is shared between httpd, webui and rest packages. It sends // incoming requests to the correct handler function @@ -54,16 +50,14 @@ func Initialize( cfg config.WebConfig, shutdownChan chan bool, mm message.Manager, - ds storage.Store, mh *msghub.Hub) { webConfig = cfg globalShutdown = shutdownChan // NewContext() will use this DataStore for the web handlers - DataStore = ds msgHub = mh - msgSvc = mm + manager = mm // Content Paths log.Infof("HTTP templates mapped to %q", cfg.TemplateDir) diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index ed092e7..16aa305 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,7 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - messages, err := ctx.MsgSvc.GetMetadata(name) + messages, err := ctx.Manager.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -94,7 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -135,7 +135,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -163,7 +163,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - r, err := ctx.MsgSvc.SourceReader(name, id) + r, err := ctx.Manager.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -198,7 +198,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -241,7 +241,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil From 9be4eec31c20bdcc22995c8cde0074fe475e86de Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 12 Mar 2018 21:13:57 -0700 Subject: [PATCH 14/82] storage: eliminate ReadBody, ReadRaw for #69 --- pkg/storage/file/fmessage.go | 61 +++++++-------------------------- pkg/storage/file/fstore_test.go | 2 +- pkg/storage/storage.go | 4 --- 3 files changed, 14 insertions(+), 53 deletions(-) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index e9df65d..ebe8540 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "net/mail" "os" "path/filepath" @@ -99,26 +98,6 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// ReadBody opens the .raw portion of a Message and returns a MIMEBody object -func (m *Message) ReadBody() (body *enmime.Envelope, err error) { - file, err := os.Open(m.rawPath()) - if err != nil { - return nil, err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - reader := bufio.NewReader(file) - mime, err := enmime.ReadEnvelope(reader) - if err != nil { - return nil, err - } - return mime, nil -} - // RawReader opens the .raw portion of a Message as an io.ReadCloser func (m *Message) RawReader() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) @@ -128,26 +107,6 @@ func (m *Message) RawReader() (reader io.ReadCloser, err error) { return file, nil } -// ReadRaw opens the .raw portion of a Message and returns it as a string -func (m *Message) ReadRaw() (raw *string, err error) { - reader, err := m.RawReader() - if err != nil { - return nil, err - } - defer func() { - if err := reader.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - bodyBytes, err := ioutil.ReadAll(bufio.NewReader(reader)) - if err != nil { - return nil, err - } - bodyString := string(bodyBytes) - return &bodyString, nil -} - // Append data to a newly opened Message, this will fail on a pre-existing Message and // after Close() is called. func (m *Message) Append(data []byte) error { @@ -195,26 +154,32 @@ func (m *Message) Close() error { } } - // Fetch headers - body, err := m.ReadBody() + // Fetch envelope. + // TODO should happen outside of datastore. + r, err := m.RawReader() + if err != nil { + return err + } + env, err := enmime.ReadEnvelope(r) + _ = r.Close() if err != nil { return err } // Only public fields are stored in gob, hence starting with capital F // Parse From address - if address, err := mail.ParseAddress(body.GetHeader("From")); err == nil { + if address, err := mail.ParseAddress(env.GetHeader("From")); err == nil { m.Ffrom = address } else { - m.Ffrom = &mail.Address{Address: body.GetHeader("From")} + m.Ffrom = &mail.Address{Address: env.GetHeader("From")} } - m.Fsubject = body.GetHeader("Subject") + m.Fsubject = env.GetHeader("Subject") // Turn the To header into a slice - if addresses, err := body.AddressList("To"); err == nil { + if addresses, err := env.AddressList("To"); err == nil { m.Fto = addresses } else { - m.Fto = []*mail.Address{{Address: body.GetHeader("To")}} + m.Fto = []*mail.Address{{Address: env.GetHeader("To")}} } // Refresh the index before adding our message diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index eb5add1..70aa168 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -337,7 +337,7 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.ReadBody() + _, err = msg.RawReader() assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 52e50cc..04f237b 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -7,8 +7,6 @@ import ( "net/mail" "sync" "time" - - "github.com/jhillyerd/enmime" ) var ( @@ -41,8 +39,6 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - ReadBody() (body *enmime.Envelope, err error) - ReadRaw() (raw *string, err error) Append(data []byte) error Close() error String() string From 2cc0da3093a56f5265133a0d75d6be1b95962452 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 13 Mar 2018 22:00:44 -0700 Subject: [PATCH 15/82] storage: More refactoring for #69 - impl Store.AddMessage - file: Use AddMessage() in tests - smtp: Switch to AddMessage - storage: Remove NewMessage, Append, Close methods --- pkg/message/message.go | 51 ++++++++++++++++++ pkg/server/smtp/handler.go | 83 ++++++++++++++++------------ pkg/server/smtp/handler_test.go | 29 ++-------- pkg/storage/file/fmessage.go | 96 +-------------------------------- pkg/storage/file/fstore.go | 58 ++++++++++++++++++++ pkg/storage/file/fstore_test.go | 42 +++++++-------- pkg/storage/retention_test.go | 33 +++++------- pkg/storage/storage.go | 5 +- pkg/storage/testing.go | 6 +++ pkg/test/storage.go | 13 ++--- 10 files changed, 208 insertions(+), 208 deletions(-) diff --git a/pkg/message/message.go b/pkg/message/message.go index fd5f25b..3994ca3 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -2,10 +2,13 @@ package message import ( + "io" + "io/ioutil" "net/mail" "time" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/storage" ) // Metadata holds information about a message, but not the content. @@ -24,3 +27,51 @@ type Message struct { Metadata Envelope *enmime.Envelope } + +// Delivery is used to add a message to storage. +type Delivery struct { + Meta Metadata + Reader io.Reader +} + +var _ storage.StoreMessage = &Delivery{} + +// Mailbox getter. +func (d *Delivery) Mailbox() string { + return d.Meta.Mailbox +} + +// ID getter. +func (d *Delivery) ID() string { + return d.Meta.ID +} + +// From getter. +func (d *Delivery) From() *mail.Address { + return d.Meta.From +} + +// To getter. +func (d *Delivery) To() []*mail.Address { + return d.Meta.To +} + +// Date getter. +func (d *Delivery) Date() time.Time { + return d.Meta.Date +} + +// Subject getter. +func (d *Delivery) Subject() string { + return d.Meta.Subject +} + +// Size getter. +func (d *Delivery) Size() int64 { + return d.Meta.Size +} + +// RawReader contains the raw content of the message. +func (d *Delivery) RawReader() (io.ReadCloser, error) { + return ioutil.NopCloser(d.Reader), nil +} diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index b1a6eb3..96fa6db 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -12,7 +12,9 @@ import ( "strings" "time" + "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -442,48 +444,61 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { - msg, err := ss.server.dataStore.NewMessage(r.localPart) - if err != nil { - ss.logError("Failed to create message for %q: %s", r.localPart, err) - return false - } - - // Generate Received header - stamp := time.Now().Format(timeStampFormat) - recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) - if err := msg.Append([]byte(recd)); err != nil { - ss.logError("Failed to write received header for %q: %s", r.localPart, err) - return false - } - - // Append lines from msgBuf - for _, line := range msgBuf { - if err := msg.Append(line); err != nil { - ss.logError("Failed to append to mailbox %v: %v", r.localPart, err) - // Should really cleanup the crap on filesystem - return false - } - } - if err := msg.Close(); err != nil { - ss.logError("Error while closing message for %v: %v", r.localPart, err) - return false - } name, err := stringutil.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false } - + buf := bytes.Buffer{} + // Generate Received header + stamp := time.Now().Format(timeStampFormat) + recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) + buf.WriteString(recd) + // Append lines from msgBuf + for _, line := range msgBuf { + buf.Write(line) + } + // TODO replace with something that only reads header? + env, err := enmime.ReadEnvelope(bytes.NewReader(buf.Bytes())) + if err != nil { + ss.logError("Failed to parse message for %q: %v", r.localPart, err) + return false + } + from, err := env.AddressList("From") + if err != nil { + ss.logError("Failed to get From address: %v", err) + return false + } + to, err := env.AddressList("To") + if err != nil { + ss.logError("Failed to get To addresses: %v", err) + return false + } + delivery := &message.Delivery{ + Meta: message.Metadata{ + Mailbox: name, + From: from[0], + To: to, + Date: time.Now(), + Subject: env.GetHeader("Subject"), + }, + Reader: bytes.NewReader(buf.Bytes()), + } + id, err := ss.server.dataStore.AddMessage(delivery) + if err != nil { + ss.logError("Failed to store message for %q: %s", r.localPart, err) + return false + } // Broadcast message information broadcast := msghub.Message{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), + ID: id, + From: delivery.From().String(), + To: stringutil.StringAddressList(delivery.To()), + Subject: delivery.Subject(), + Date: delivery.Date(), + Size: delivery.Size(), } ss.server.msgHub.Dispatch(broadcast) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index ca9ff2e..94b0d48 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -8,7 +8,6 @@ import ( "log" "net" - "net/mail" "net/textproto" "os" "testing" @@ -141,18 +140,7 @@ func TestReadyState(t *testing.T) { // Test commands in MAIL state func TestMailState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - msg1 := &storage.MockMessage{} - mds.On("NewMessage", "u1").Return(msg1, nil) - msg1.On("ID").Return("") - msg1.On("From").Return(&mail.Address{}) - msg1.On("To").Return(make([]*mail.Address, 0)) - msg1.On("Date").Return(time.Time{}) - msg1.On("Subject").Return("") - msg1.On("Size").Return(0) - msg1.On("Close").Return(nil) - + mds := test.NewStore() server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -214,7 +202,7 @@ func TestMailState(t *testing.T) { {"MAIL FROM:", 250}, {"RCPT TO:", 250}, {"DATA", 354}, - {".", 250}, + {".", 451}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -253,18 +241,7 @@ func TestMailState(t *testing.T) { // Test commands in DATA state func TestDataState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - msg1 := &storage.MockMessage{} - mds.On("NewMessage", "u1").Return(msg1, nil) - msg1.On("ID").Return("") - msg1.On("From").Return(&mail.Address{}) - msg1.On("To").Return(make([]*mail.Address, 0)) - msg1.On("Date").Return(time.Time{}) - msg1.On("Subject").Return("") - msg1.On("Size").Return(0) - msg1.On("Close").Return(nil) - + mds := test.NewStore() server, logbuf, teardown := setupSMTPServer(mds) defer teardown() diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index ebe8540..cf06d5c 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -2,16 +2,13 @@ package file import ( "bufio" - "fmt" "io" "net/mail" "os" "path/filepath" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Message implements Message and contains a little bit of data about a @@ -33,7 +30,7 @@ type Message struct { // newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *mbox) newMessage() (storage.StoreMessage, error) { +func (mb *mbox) newMessage() (*Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -84,11 +81,6 @@ func (m *Message) Subject() string { return m.Fsubject } -// String returns a string in the form: "Subject()" from From() -func (m *Message) String() string { - return fmt.Sprintf("\"%v\" from %v", m.Fsubject, m.Ffrom) -} - // Size returns the size of the Message on disk in bytes func (m *Message) Size() int64 { return m.Fsize @@ -106,89 +98,3 @@ func (m *Message) RawReader() (reader io.ReadCloser, err error) { } return file, nil } - -// Append data to a newly opened Message, this will fail on a pre-existing Message and -// after Close() is called. -func (m *Message) Append(data []byte) error { - // Prevent Appending to a pre-existing Message - if !m.writable { - return storage.ErrNotWritable - } - // Open file for writing if we haven't yet - if m.writer == nil { - // Ensure mailbox directory exists - if err := m.mailbox.createDir(); err != nil { - return err - } - file, err := os.Create(m.rawPath()) - if err != nil { - // Set writable false just in case something calls me a million times - m.writable = false - return err - } - m.writerFile = file - m.writer = bufio.NewWriter(file) - } - _, err := m.writer.Write(data) - m.Fsize += int64(len(data)) - return err -} - -// Close this Message for writing - no more data may be Appended. Close() will also -// trigger the creation of the .gob file. -func (m *Message) Close() error { - // nil out the writer fields so they can't be used - writer := m.writer - writerFile := m.writerFile - m.writer = nil - m.writerFile = nil - - if writer != nil { - if err := writer.Flush(); err != nil { - return err - } - } - if writerFile != nil { - if err := writerFile.Close(); err != nil { - return err - } - } - - // Fetch envelope. - // TODO should happen outside of datastore. - r, err := m.RawReader() - if err != nil { - return err - } - env, err := enmime.ReadEnvelope(r) - _ = r.Close() - if err != nil { - return err - } - - // Only public fields are stored in gob, hence starting with capital F - // Parse From address - if address, err := mail.ParseAddress(env.GetHeader("From")); err == nil { - m.Ffrom = address - } else { - m.Ffrom = &mail.Address{Address: env.GetHeader("From")} - } - m.Fsubject = env.GetHeader("Subject") - - // Turn the To header into a slice - if addresses, err := env.AddressList("To"); err == nil { - m.Fto = addresses - } else { - m.Fto = []*mail.Address{{Address: env.GetHeader("To")}} - } - - // Refresh the index before adding our message - err = m.mailbox.readIndex() - if err != nil { - return err - } - - // Made it this far without errors, add it to the index - m.mailbox.messages = append(m.mailbox.messages, m) - return m.mailbox.writeIndex() -} diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 3e5f9bb..c9ecfee 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -74,6 +74,64 @@ func New(cfg config.DataStoreConfig) storage.Store { return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } +// AddMessage adds a message to the specified mailbox. +func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { + r, err := m.RawReader() + if err != nil { + return "", err + } + mb, err := fs.mbox(m.Mailbox()) + if err != nil { + return "", err + } + // Create a new message. + fm, err := mb.newMessage() + if err != nil { + return "", err + } + // Ensure mailbox directory exists. + if err := mb.createDir(); err != nil { + return "", err + } + // Write the message content + file, err := os.Create(fm.rawPath()) + if err != nil { + return "", err + } + w := bufio.NewWriter(file) + size, err := io.Copy(w, r) + if err != nil { + // Try to remove the file + _ = file.Close() + _ = os.Remove(fm.rawPath()) + return "", err + } + _ = r.Close() + if err := w.Flush(); err != nil { + // Try to remove the file + _ = file.Close() + _ = os.Remove(fm.rawPath()) + return "", err + } + if err := file.Close(); err != nil { + // Try to remove the file + _ = os.Remove(fm.rawPath()) + return "", err + } + // Update the index. + fm.Fdate = m.Date() + fm.Ffrom = m.From() + fm.Fsize = size + fm.Fsubject = m.Subject() + mb.messages = append(mb.messages, fm) + if err := mb.writeIndex(); err != nil { + // Try to remove the file + _ = os.Remove(fm.rawPath()) + return "", err + } + return fm.Fid, nil +} + // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 70aa168..4b2f287 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -6,12 +6,15 @@ import ( "io" "io/ioutil" "log" + "net/mail" "os" "path/filepath" + "strings" "testing" "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/stretchr/testify/assert" ) @@ -480,32 +483,25 @@ func setupDataStore(cfg config.DataStoreConfig) (*Store, *bytes.Buffer) { // deliverMessage creates and delivers a message to the specific mailbox, returning // the size of the generated message. -func deliverMessage(ds *Store, mbName string, subject string, - date time.Time) (id string, size int64) { - // Build fake SMTP message for delivery - testMsg := make([]byte, 0, 300) - testMsg = append(testMsg, []byte("To: somebody@host\r\n")...) - testMsg = append(testMsg, []byte("From: somebodyelse@host\r\n")...) - testMsg = append(testMsg, []byte(fmt.Sprintf("Subject: %s\r\n", subject))...) - testMsg = append(testMsg, []byte("\r\n")...) - testMsg = append(testMsg, []byte("Test Body\r\n")...) - - // Create message object - id = generateID(date) - msg, err := ds.NewMessage(mbName) +func deliverMessage(ds *Store, mbName string, subject string, date time.Time) (string, int64) { + // Build message for delivery + meta := message.Metadata{ + Mailbox: mbName, + To: []*mail.Address{{Name: "", Address: "somebody@host"}}, + From: &mail.Address{Name: "", Address: "somebodyelse@host"}, + Subject: subject, + Date: date, + } + testMsg := fmt.Sprintf("To: %s\r\nFrom: %s\r\nSubject: %s\r\n\r\nTest Body\r\n", + meta.To[0].Address, meta.From.Address, subject) + delivery := &message.Delivery{ + Meta: meta, + Reader: ioutil.NopCloser(strings.NewReader(testMsg)), + } + id, err := ds.AddMessage(delivery) if err != nil { panic(err) } - fmsg := msg.(*Message) - fmsg.Fdate = date - fmsg.Fid = id - if err = msg.Append(testMsg); err != nil { - panic(err) - } - if err = msg.Close(); err != nil { - panic(err) - } - return id, int64(len(testMsg)) } diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index f6ed828..5a0d07e 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -13,24 +13,18 @@ import ( func TestDoRetentionScan(t *testing.T) { ds := test.NewStore() // Mockup some different aged messages (num is in hours) - new1 := mockMessage(0) - new2 := mockMessage(1) - new3 := mockMessage(2) - old1 := mockMessage(4) - old2 := mockMessage(12) - old3 := mockMessage(24) - ds.AddMessage("mb1", new1) - new1.On("Mailbox").Return("mb1") - ds.AddMessage("mb1", old1) - old1.On("Mailbox").Return("mb1") - ds.AddMessage("mb1", old2) - old2.On("Mailbox").Return("mb1") - ds.AddMessage("mb2", old3) - old3.On("Mailbox").Return("mb2") - ds.AddMessage("mb2", new2) - new2.On("Mailbox").Return("mb2") - ds.AddMessage("mb3", new3) - new3.On("Mailbox").Return("mb3") + new1 := mockMessage("mb1", 0) + new2 := mockMessage("mb2", 1) + new3 := mockMessage("mb3", 2) + old1 := mockMessage("mb1", 4) + old2 := mockMessage("mb1", 12) + old3 := mockMessage("mb2", 24) + ds.AddMessage(new1) + ds.AddMessage(old1) + ds.AddMessage(old2) + ds.AddMessage(old3) + ds.AddMessage(new2) + ds.AddMessage(new3) // Test 4 hour retention cfg := config.DataStoreConfig{ RetentionMinutes: 239, @@ -56,8 +50,9 @@ func TestDoRetentionScan(t *testing.T) { } // Make a MockMessage of a specific age -func mockMessage(ageHours int) *storage.MockMessage { +func mockMessage(mailbox string, ageHours int) *storage.MockMessage { msg := &storage.MockMessage{} + msg.On("Mailbox").Return(mailbox) msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) msg.On("Delete").Return(nil) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 04f237b..8f6783a 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -19,6 +19,8 @@ var ( // Store is the interface Inbucket uses to interact with storage implementations. type Store interface { + // AddMessage stores the message, message ID and Size will be ignored. + AddMessage(message StoreMessage) (id string, err error) GetMessage(mailbox, id string) (StoreMessage, error) GetMessages(mailbox string) ([]StoreMessage, error) PurgeMessages(mailbox string) error @@ -39,8 +41,5 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - Append(data []byte) error - Close() error - String() string Size() int64 } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 9defa55..b987477 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -15,6 +15,12 @@ type MockDataStore struct { mock.Mock } +// AddMessage mock function +func (m *MockDataStore) AddMessage(message StoreMessage) (string, error) { + args := m.Called(message) + return args.String(0), args.Error(1) +} + // GetMessage mock function func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { args := m.Called(name, id) diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 92195ff..52c9b00 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -23,9 +23,11 @@ func NewStore() *StoreStub { } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(mailbox string, m storage.StoreMessage) { - msgs := s.mailboxes[mailbox] - s.mailboxes[mailbox] = append(msgs, m) +func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { + mb := m.Mailbox() + msgs := s.mailboxes[mb] + s.mailboxes[mb] = append(msgs, m) + return m.ID(), nil } // GetMessage gets a message by ID from the specified mailbox. @@ -80,11 +82,6 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e return nil } -// NewMessage is temproary until #69 MessageData refactor -func (s *StoreStub) NewMessage(mailbox string) (storage.StoreMessage, error) { - return nil, nil -} - // LockFor mock function returns a new RWMutex, never errors. // TODO(#69) remove func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { From 519779b7baa45a5cc5e094c9c94a1500e4ab2982 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 14 Mar 2018 21:05:59 -0700 Subject: [PATCH 16/82] storage: eliminate mocks, closes #80 --- pkg/rest/testutils_test.go | 27 ------ pkg/storage/retention_test.go | 30 +++---- pkg/storage/testing.go | 152 ---------------------------------- 3 files changed, 16 insertions(+), 193 deletions(-) delete mode 100644 pkg/storage/testing.go diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index cfddede..23996a5 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -9,12 +9,10 @@ import ( "net/mail" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/storage" ) type InputMessageData struct { @@ -26,31 +24,6 @@ type InputMessageData struct { HTML, Text string } -func (d *InputMessageData) MockMessage() *storage.MockMessage { - from, _ := mail.ParseAddress(d.From) - to := make([]*mail.Address, len(d.To)) - for i, a := range d.To { - to[i], _ = mail.ParseAddress(a) - } - msg := &storage.MockMessage{} - msg.On("ID").Return(d.ID) - msg.On("From").Return(from) - msg.On("To").Return(to) - msg.On("Subject").Return(d.Subject) - msg.On("Date").Return(d.Date) - msg.On("Size").Return(d.Size) - gomsg := &mail.Message{ - Header: d.Header, - } - msg.On("ReadHeader").Return(gomsg, nil) - body := &enmime.Envelope{ - Text: d.Text, - HTML: d.HTML, - } - msg.On("ReadBody").Return(body, nil) - return msg -} - // isJSONStringEqual is a utility function to return a nicely formatted message when // comparing a string to a value received from a JSON map. func isJSONStringEqual(key, expected string, received interface{}) (message string, ok bool) { diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index 5a0d07e..a49cc59 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -13,12 +14,12 @@ import ( func TestDoRetentionScan(t *testing.T) { ds := test.NewStore() // Mockup some different aged messages (num is in hours) - new1 := mockMessage("mb1", 0) - new2 := mockMessage("mb2", 1) - new3 := mockMessage("mb3", 2) - old1 := mockMessage("mb1", 4) - old2 := mockMessage("mb1", 12) - old3 := mockMessage("mb2", 24) + new1 := stubMessage("mb1", 0) + new2 := stubMessage("mb2", 1) + new3 := stubMessage("mb3", 2) + old1 := stubMessage("mb1", 4) + old2 := stubMessage("mb1", 12) + old3 := stubMessage("mb2", 24) ds.AddMessage(new1) ds.AddMessage(old1) ds.AddMessage(old2) @@ -49,12 +50,13 @@ func TestDoRetentionScan(t *testing.T) { } } -// Make a MockMessage of a specific age -func mockMessage(mailbox string, ageHours int) *storage.MockMessage { - msg := &storage.MockMessage{} - msg.On("Mailbox").Return(mailbox) - msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) - msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) - msg.On("Delete").Return(nil) - return msg +// stubMessage creates a message stub of a specific age +func stubMessage(mailbox string, ageHours int) storage.StoreMessage { + return &message.Delivery{ + Meta: message.Metadata{ + Mailbox: mailbox, + ID: fmt.Sprintf("MSG[age=%vh]", ageHours), + Date: time.Now().Add(time.Duration(ageHours*-1) * time.Hour), + }, + } } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go deleted file mode 100644 index b987477..0000000 --- a/pkg/storage/testing.go +++ /dev/null @@ -1,152 +0,0 @@ -package storage - -import ( - "io" - "net/mail" - "sync" - "time" - - "github.com/jhillyerd/enmime" - "github.com/stretchr/testify/mock" -) - -// MockDataStore is a shared mock for unit testing -type MockDataStore struct { - mock.Mock -} - -// AddMessage mock function -func (m *MockDataStore) AddMessage(message StoreMessage) (string, error) { - args := m.Called(message) - return args.String(0), args.Error(1) -} - -// GetMessage mock function -func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { - args := m.Called(name, id) - return args.Get(0).(StoreMessage), args.Error(1) -} - -// GetMessages mock function -func (m *MockDataStore) GetMessages(name string) ([]StoreMessage, error) { - args := m.Called(name) - return args.Get(0).([]StoreMessage), args.Error(1) -} - -// RemoveMessage mock function -func (m *MockDataStore) RemoveMessage(name, id string) error { - args := m.Called(name, id) - return args.Error(0) -} - -// PurgeMessages mock function -func (m *MockDataStore) PurgeMessages(name string) error { - args := m.Called(name) - return args.Error(0) -} - -// LockFor mock function returns a new RWMutex, never errors. -func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { - return &sync.RWMutex{}, nil -} - -// NewMessage temporary for #69 -func (m *MockDataStore) NewMessage(mailbox string) (StoreMessage, error) { - args := m.Called(mailbox) - return args.Get(0).(StoreMessage), args.Error(1) -} - -// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it -// continues to return true. -func (m *MockDataStore) VisitMailboxes(f func([]StoreMessage) (cont bool)) error { - return nil -} - -// MockMessage is a shared mock for unit testing -type MockMessage struct { - mock.Mock -} - -// Mailbox mock function -func (m *MockMessage) Mailbox() string { - args := m.Called() - return args.String(0) -} - -// ID mock function -func (m *MockMessage) ID() string { - args := m.Called() - return args.String(0) -} - -// From mock function -func (m *MockMessage) From() *mail.Address { - args := m.Called() - return args.Get(0).(*mail.Address) -} - -// To mock function -func (m *MockMessage) To() []*mail.Address { - args := m.Called() - return args.Get(0).([]*mail.Address) -} - -// Date mock function -func (m *MockMessage) Date() time.Time { - args := m.Called() - return args.Get(0).(time.Time) -} - -// Subject mock function -func (m *MockMessage) Subject() string { - args := m.Called() - return args.String(0) -} - -// ReadHeader mock function -func (m *MockMessage) ReadHeader() (msg *mail.Message, err error) { - args := m.Called() - return args.Get(0).(*mail.Message), args.Error(1) -} - -// ReadBody mock function -func (m *MockMessage) ReadBody() (body *enmime.Envelope, err error) { - args := m.Called() - return args.Get(0).(*enmime.Envelope), args.Error(1) -} - -// ReadRaw mock function -func (m *MockMessage) ReadRaw() (raw *string, err error) { - args := m.Called() - return args.Get(0).(*string), args.Error(1) -} - -// RawReader mock function -func (m *MockMessage) RawReader() (reader io.ReadCloser, err error) { - args := m.Called() - return args.Get(0).(io.ReadCloser), args.Error(1) -} - -// Size mock function -func (m *MockMessage) Size() int64 { - args := m.Called() - return int64(args.Int(0)) -} - -// Append mock function -func (m *MockMessage) Append(data []byte) error { - // []byte arg seems to mess up testify/mock - return nil -} - -// Close mock function -func (m *MockMessage) Close() error { - args := m.Called() - return args.Error(0) -} - -// String mock function -func (m *MockMessage) String() string { - args := m.Called() - return args.String(0) -} From 5e13e5076301a9b2eaaaa95a8e4b35df93d39b0f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 14 Mar 2018 22:51:40 -0700 Subject: [PATCH 17/82] test: Start work on test suite for #82 - smtp: Tidy up []byte/buffer/string use in delivery #69 --- pkg/server/smtp/handler.go | 60 ++++++++-------------- pkg/storage/file/fstore.go | 1 + pkg/storage/file/fstore_test.go | 14 ++++++ pkg/storage/storage.go | 2 - pkg/test/storage_suite.go | 89 +++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 42 deletions(-) create mode 100644 pkg/test/storage_suite.go diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 96fa6db..d9dad39 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -352,7 +352,6 @@ func (ss *Session) mailHandler(cmd string, arg string) { func (ss *Session) dataHandler() { recipients := make([]recipientDetails, 0, ss.recipients.Len()) // Get a Mailbox and a new Message for each recipient - msgSize := 0 if ss.server.storeMessages { for e := ss.recipients.Front(); e != nil; e = e.Next() { recip := e.Value.(string) @@ -373,11 +372,9 @@ func (ss *Session) dataHandler() { } ss.send("354 Start mail input; end with .") - var lineBuf bytes.Buffer - msgBuf := make([][]byte, 0, 1024) + msgBuf := &bytes.Buffer{} for { - lineBuf.Reset() - err := ss.readByteLine(&lineBuf) + lineBuf, err := ss.readByteLine() if err != nil { if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { @@ -388,9 +385,7 @@ func (ss *Session) dataHandler() { ss.enterState(QUIT) return } - line := lineBuf.Bytes() - // ss.logTrace("DATA: %q", line) - if string(line) == ".\r\n" || string(line) == ".\n" { + if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete if ss.server.storeMessages { // Create a message for each valid recipient @@ -405,7 +400,7 @@ func (ss *Session) dataHandler() { return } mu.Lock() - ok := ss.deliverMessage(r, msgBuf) + ok := ss.deliverMessage(r, msgBuf.Bytes()) mu.Unlock() if ok { expReceivedTotal.Add(1) @@ -420,47 +415,34 @@ func (ss *Session) dataHandler() { expReceivedTotal.Add(1) } ss.send("250 Mail accepted for delivery") - ss.logInfo("Message size %v bytes", msgSize) + ss.logInfo("Message size %v bytes", msgBuf.Len()) ss.reset() return } // SMTP RFC says remove leading periods from input - if len(line) > 0 && line[0] == '.' { - line = line[1:] + if len(lineBuf) > 0 && lineBuf[0] == '.' { + lineBuf = lineBuf[1:] } - // Second append copies line/lineBuf so we can reuse it - msgBuf = append(msgBuf, append([]byte{}, line...)) - msgSize += len(line) - if msgSize > ss.server.maxMessageBytes { + msgBuf.Write(lineBuf) + if msgBuf.Len() > ss.server.maxMessageBytes { // Max message size exceeded ss.send("552 Maximum message size exceeded") ss.logWarn("Max message size exceeded while in DATA") ss.reset() - // Should really cleanup the crap on filesystem (after issue #23) return } } // end for } // deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { +func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { name, err := stringutil.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false } - buf := bytes.Buffer{} - // Generate Received header - stamp := time.Now().Format(timeStampFormat) - recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) - buf.WriteString(recd) - // Append lines from msgBuf - for _, line := range msgBuf { - buf.Write(line) - } // TODO replace with something that only reads header? - env, err := enmime.ReadEnvelope(bytes.NewReader(buf.Bytes())) + env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { ss.logError("Failed to parse message for %q: %v", r.localPart, err) return false @@ -475,6 +457,10 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) ss.logError("Failed to get To addresses: %v", err) return false } + // Generate Received header. + stamp := time.Now().Format(timeStampFormat) + recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ Mailbox: name, @@ -483,14 +469,14 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) Date: time.Now(), Subject: env.GetHeader("Subject"), }, - Reader: bytes.NewReader(buf.Bytes()), + Reader: io.MultiReader(recd, bytes.NewReader(content)), } id, err := ss.server.dataStore.AddMessage(delivery) if err != nil { ss.logError("Failed to store message for %q: %s", r.localPart, err) return false } - // Broadcast message information + // Broadcast message information. broadcast := msghub.Message{ Mailbox: name, ID: id, @@ -501,7 +487,6 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) Size: delivery.Size(), } ss.server.msgHub.Dispatch(broadcast) - return true } @@ -535,16 +520,11 @@ func (ss *Session) send(msg string) { // readByteLine reads a line of input into the provided buffer. Does // not reset the Buffer - please do so prior to calling. -func (ss *Session) readByteLine(buf io.Writer) error { +func (ss *Session) readByteLine() ([]byte, error) { if err := ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { - return err + return nil, err } - line, err := ss.reader.ReadBytes('\n') - if err != nil { - return err - } - _, err = buf.Write(line) - return err + return ss.reader.ReadBytes('\n') } // Reads a line of input diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index c9ecfee..b41b5d3 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -121,6 +121,7 @@ func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { // Update the index. fm.Fdate = m.Date() fm.Ffrom = m.From() + fm.Fto = m.To() fm.Fsize = size fm.Fsubject = m.Subject() mb.messages = append(mb.messages, fm) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 4b2f287..94380d9 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -16,9 +16,23 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" "github.com/stretchr/testify/assert" ) +// TestSuite runs storage package test suite on file store. +func TestSuite(t *testing.T) { + ds, logbuf := setupDataStore(config.DataStoreConfig{}) + defer teardownDataStore(ds) + test.StoreSuite(t, ds) + if t.Failed() { + // Wait for handler to finish logging. + time.Sleep(2 * time.Second) + // Dump buffered log data if there was a failure. + _, _ = io.Copy(os.Stderr, logbuf) + } +} + // Test directory structure created by filestore func TestFSDirStructure(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8f6783a..62d4972 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,8 +28,6 @@ type Store interface { VisitMailboxes(f func([]StoreMessage) (cont bool)) error // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) - // NewMessage is temproary until #69 MessageData refactor - NewMessage(mailbox string) (StoreMessage, error) } // StoreMessage represents a message to be stored, or returned from a storage implementation. diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go new file mode 100644 index 0000000..fd71774 --- /dev/null +++ b/pkg/test/storage_suite.go @@ -0,0 +1,89 @@ +package test + +import ( + "net/mail" + "strings" + "testing" + "time" + + "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// StoreSuite runs a set of general tests on the provided Store +func StoreSuite(t *testing.T, store storage.Store) { + testCases := []struct { + name string + test func(*testing.T, storage.Store) + }{ + {"metadata", testMetadata}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.test(t, store) + }) + } +} + +func testMetadata(t *testing.T, ds storage.Store) { + // Store a message + mailbox := "testmailbox" + from := &mail.Address{Name: "From Person", Address: "from@person.com"} + to := []*mail.Address{ + {Name: "One Person", Address: "one@a.person.com"}, + {Name: "Two Person", Address: "two@b.person.com"}, + } + date := time.Now() + subject := "fantastic test subject line" + content := "doesn't matter" + delivery := &message.Delivery{ + Meta: message.Metadata{ + // ID and Size will be determined by the Store + Mailbox: mailbox, + From: from, + To: to, + Date: date, + Subject: subject, + }, + Reader: strings.NewReader(content), + } + id, err := ds.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + if id == "" { + t.Fatal("Expected AddMessage() to return non-empty ID string") + } + // Retrieve and validate the message + sm, err := ds.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + if sm.Mailbox() != mailbox { + t.Errorf("got mailbox %q, want: %q", sm.Mailbox(), mailbox) + } + if sm.ID() != id { + t.Errorf("got id %q, want: %q", sm.ID(), id) + } + if *sm.From() != *from { + t.Errorf("got from %v, want: %v", sm.From(), from) + } + if len(sm.To()) != len(to) { + t.Errorf("got len(to) = %v, want: %v", len(sm.To()), len(to)) + } else { + for i, got := range sm.To() { + if *to[i] != *got { + t.Errorf("got to[%v] %v, want: %v", i, got, to[i]) + } + } + } + if !sm.Date().Equal(date) { + t.Errorf("got date %v, want: %v", sm.Date(), date) + } + if sm.Subject() != subject { + t.Errorf("got subject %q, want: %q", sm.Subject(), subject) + } + if sm.Size() != int64(len(content)) { + t.Errorf("got size %v, want: %v", sm.Size(), len(content)) + } +} From 9b3d3c2ea821586d669cfbd06f3b346f2531c2d2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Fri, 16 Mar 2018 22:05:07 -0700 Subject: [PATCH 18/82] test: Finish initial storage test suite, closes #82 --- Makefile | 5 +- pkg/storage/file/fstore_test.go | 235 +---------------------------- pkg/test/storage_suite.go | 257 +++++++++++++++++++++++++++++++- 3 files changed, 260 insertions(+), 237 deletions(-) diff --git a/Makefile b/Makefile index 821c674..eacf4ba 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SHELL = /bin/sh SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*") PKGS := $(shell go list ./... | grep -v /vendor/) -.PHONY: all build clean fmt lint simplify test +.PHONY: all build clean fmt lint reflex simplify test commands = client inbucket @@ -34,3 +34,6 @@ lint: @test -z "$(shell gofmt -l . | tee /dev/stderr)" || echo "[WARN] Fix formatting issues with 'make fmt'" @golint -set_exit_status $(PKGS) @go vet $(PKGS) + +reflex: + reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./...' diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 94380d9..18b1fe7 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -22,15 +22,13 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - test.StoreSuite(t, ds) - if t.Failed() { - // Wait for handler to finish logging. - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure. - _, _ = io.Copy(os.Stderr, logbuf) - } + test.StoreSuite(t, func() (storage.Store, func(), error) { + ds, _ := setupDataStore(config.DataStoreConfig{}) + destroy := func() { + teardownDataStore(ds) + } + return ds, destroy, nil + }) } // Test directory structure created by filestore @@ -111,225 +109,6 @@ func TestFSDirStructure(t *testing.T) { } } -// TestFSVisitMailboxes tests VisitMailboxes -func TestFSVisitMailboxes(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} - for _, name := range boxes { - // Create day old message - date := time.Now().Add(-24 * time.Hour) - deliverMessage(ds, name, "Old Message", date) - - // Create current message - date = time.Now() - deliverMessage(ds, name, "New Message", date) - } - - seen := 0 - err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { - seen++ - count := len(messages) - if count != 2 { - t.Errorf("got: %v messages, want: 2", count) - } - return true - }) - assert.Nil(t, err) - assert.Equal(t, 5, seen) - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test delivering several messages to the same mailbox, meanwhile querying its -// contents with a new mailbox object each time -func TestFSDeliverMany(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for i, subj := range subjects { - // Check number of messages - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, i, len(msgs), "Expected %v message(s), but got %v", i, len(msgs)) - - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Confirm delivery order - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test deleting messages -func TestFSDelete(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for _, subj := range subjects { - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Delete a couple messages - err = ds.RemoveMessage(mbName, msgs[1].ID()) - if err != nil { - t.Fatal(err) - } - err = ds.RemoveMessage(mbName, msgs[3].ID()) - if err != nil { - t.Fatal(err) - } - - // Confirm deletion - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - subjects = []string{"alpha", "charlie", "echo"} - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - // Try appending one more - deliverMessage(ds, mbName, "foxtrot", time.Now()) - - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - subjects = []string{"alpha", "charlie", "echo", "foxtrot"} - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test purging a mailbox -func TestFSPurge(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for _, subj := range subjects { - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Purge mailbox - err = ds.PurgeMessages(mbName) - assert.Nil(t, err) - - // Confirm deletion - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - assert.Equal(t, len(msgs), 0, "Expected mailbox to have zero messages, got %v", len(msgs)) - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test message size calculation -func TestFSSize(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"a", "br", "much longer than the others"} - sentIds := make([]string, len(subjects)) - sentSizes := make([]int64, len(subjects)) - - for i, subj := range subjects { - // Add a message - id, size := deliverMessage(ds, mbName, subj, time.Now()) - sentIds[i] = id - sentSizes[i] = size - } - - for i, id := range sentIds { - msg, err := ds.GetMessage(mbName, id) - assert.Nil(t, err) - - expect := sentSizes[i] - size := msg.Size() - assert.Equal(t, expect, size, "Expected size of %v, got %v", expect, size) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - // Test missing files func TestFSMissing(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index fd71774..80e8626 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -1,6 +1,9 @@ package test import ( + "bytes" + "fmt" + "io/ioutil" "net/mail" "strings" "testing" @@ -10,23 +13,37 @@ import ( "github.com/jhillyerd/inbucket/pkg/storage" ) -// StoreSuite runs a set of general tests on the provided Store -func StoreSuite(t *testing.T, store storage.Store) { +// StoreFactory returns a new store for the test suite. +type StoreFactory func() (store storage.Store, destroy func(), err error) + +// StoreSuite runs a set of general tests on the provided Store. +func StoreSuite(t *testing.T, factory StoreFactory) { testCases := []struct { name string test func(*testing.T, storage.Store) }{ {"metadata", testMetadata}, + {"content", testContent}, + {"delivery order", testDeliveryOrder}, + {"size", testSize}, + {"delete", testDelete}, + {"purge", testPurge}, + {"visit mailboxes", testVisitMailboxes}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + store, destroy, err := factory() + if err != nil { + t.Fatal(err) + } tc.test(t, store) + destroy() }) } } -func testMetadata(t *testing.T, ds storage.Store) { - // Store a message +// testMetadata verifies message metadata is stored and retrieved correctly. +func testMetadata(t *testing.T, store storage.Store) { mailbox := "testmailbox" from := &mail.Address{Name: "From Person", Address: "from@person.com"} to := []*mail.Address{ @@ -38,7 +55,7 @@ func testMetadata(t *testing.T, ds storage.Store) { content := "doesn't matter" delivery := &message.Delivery{ Meta: message.Metadata{ - // ID and Size will be determined by the Store + // ID and Size will be determined by the Store. Mailbox: mailbox, From: from, To: to, @@ -47,15 +64,15 @@ func testMetadata(t *testing.T, ds storage.Store) { }, Reader: strings.NewReader(content), } - id, err := ds.AddMessage(delivery) + id, err := store.AddMessage(delivery) if err != nil { t.Fatal(err) } if id == "" { t.Fatal("Expected AddMessage() to return non-empty ID string") } - // Retrieve and validate the message - sm, err := ds.GetMessage(mailbox, id) + // Retrieve and validate the message. + sm, err := store.GetMessage(mailbox, id) if err != nil { t.Fatal(err) } @@ -87,3 +104,227 @@ func testMetadata(t *testing.T, ds storage.Store) { t.Errorf("got size %v, want: %v", sm.Size(), len(content)) } } + +// testContent generates some binary content and makes sure it is correctly retrieved. +func testContent(t *testing.T, store storage.Store) { + content := make([]byte, 5000) + for i := 0; i < len(content); i++ { + content[i] = byte(i % 256) + } + mailbox := "testmailbox" + from := &mail.Address{Name: "From Person", Address: "from@person.com"} + to := []*mail.Address{ + {Name: "One Person", Address: "one@a.person.com"}, + } + date := time.Now() + subject := "fantastic test subject line" + delivery := &message.Delivery{ + Meta: message.Metadata{ + // ID and Size will be determined by the Store. + Mailbox: mailbox, + From: from, + To: to, + Date: date, + Subject: subject, + }, + Reader: bytes.NewReader(content), + } + id, err := store.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + // Get and check. + m, err := store.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + r, err := m.RawReader() + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if len(got) != len(content) { + t.Errorf("Got len(content) == %v, want: %v", len(got), len(content)) + } + errors := 0 + for i, b := range got { + if b != content[i] { + t.Errorf("Got content[%v] == %v, want: %v", i, b, content[i]) + errors++ + } + if errors > 5 { + t.Fatalf("Too many content errors, aborting test.") + break + } + } +} + +// testDeliveryOrder delivers several messages to the same mailbox, meanwhile querying its contents +// with a new GetMessages call each cycle. +func testDeliveryOrder(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for i, subj := range subjects { + // Check mailbox count. + getAndCountMessages(t, store, mailbox, i) + deliverMessage(t, store, mailbox, subj, time.Now()) + } + // Confirm delivery order. + msgs := getAndCountMessages(t, store, mailbox, 5) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } +} + +// testSize verifies message contnet size metadata values. +func testSize(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"a", "br", "much longer than the others"} + sentIds := make([]string, len(subjects)) + sentSizes := make([]int64, len(subjects)) + for i, subj := range subjects { + id, size := deliverMessage(t, store, mailbox, subj, time.Now()) + sentIds[i] = id + sentSizes[i] = size + } + for i, id := range sentIds { + msg, err := store.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + want := sentSizes[i] + got := msg.Size() + if got != want { + t.Errorf("Got size %v, want: %v", got, want) + } + } +} + +// testDelete creates and deletes some messages. +func testDelete(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for _, subj := range subjects { + deliverMessage(t, store, mailbox, subj, time.Now()) + } + msgs := getAndCountMessages(t, store, mailbox, len(subjects)) + // Delete a couple messages. + err := store.RemoveMessage(mailbox, msgs[1].ID()) + if err != nil { + t.Fatal(err) + } + err = store.RemoveMessage(mailbox, msgs[3].ID()) + if err != nil { + t.Fatal(err) + } + // Confirm deletion. + subjects = []string{"alpha", "charlie", "echo"} + msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } + // Try appending one more. + deliverMessage(t, store, mailbox, "foxtrot", time.Now()) + subjects = []string{"alpha", "charlie", "echo", "foxtrot"} + msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } +} + +// testPurge makes sure mailboxes can be purged. +func testPurge(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for _, subj := range subjects { + deliverMessage(t, store, mailbox, subj, time.Now()) + } + getAndCountMessages(t, store, mailbox, len(subjects)) + // Purge and verify. + err := store.PurgeMessages(mailbox) + if err != nil { + t.Fatal(err) + } + getAndCountMessages(t, store, mailbox, 0) +} + +// testVisitMailboxes creates some mailboxes and confirms the VisitMailboxes method visits all of +// them. +func testVisitMailboxes(t *testing.T, ds storage.Store) { + boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} + for _, name := range boxes { + deliverMessage(t, ds, name, "Old Message", time.Now().Add(-24*time.Hour)) + deliverMessage(t, ds, name, "New Message", time.Now()) + } + seen := 0 + err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { + seen++ + count := len(messages) + if count != 2 { + t.Errorf("got: %v messages, want: 2", count) + } + return true + }) + if err != nil { + t.Error(err) + } + if seen != 5 { + t.Errorf("saw %v messages in total, want: 5", seen) + } +} + +// deliverMessage creates and delivers a message to the specific mailbox, returning the size of the +// generated message. +func deliverMessage( + t *testing.T, + store storage.Store, + mailbox string, + subject string, + date time.Time, +) (string, int64) { + t.Helper() + meta := message.Metadata{ + Mailbox: mailbox, + To: []*mail.Address{{Name: "Some Body", Address: "somebody@host"}}, + From: &mail.Address{Name: "Some B. Else", Address: "somebodyelse@host"}, + Subject: subject, + Date: date, + } + testMsg := fmt.Sprintf("To: %s\r\nFrom: %s\r\nSubject: %s\r\n\r\nTest Body\r\n", + meta.To[0].Address, meta.From.Address, subject) + delivery := &message.Delivery{ + Meta: meta, + Reader: ioutil.NopCloser(strings.NewReader(testMsg)), + } + id, err := store.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + return id, int64(len(testMsg)) +} + +// getAndCountMessages is a test helper that expects to receive count messages or fails the test, it +// also checks return error. +func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.StoreMessage { + t.Helper() + msgs, err := s.GetMessages(mailbox) + if err != nil { + t.Fatalf("Failed to GetMessages for %q: %v", mailbox, err) + } + if len(msgs) != count { + t.Errorf("Got %v messages, want: %v", len(msgs), count) + } + return msgs +} From d132efd6fadc73a38ae947330cb6acde8b988a72 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 09:18:28 -0700 Subject: [PATCH 19/82] policy: Create new policy package for #84 --- pkg/policy/address.go | 204 +++++++++++++++++++++++++++++ pkg/policy/address_test.go | 138 ++++++++++++++++++++ pkg/rest/apiv1_controller.go | 11 +- pkg/rest/socketv1_controller.go | 4 +- pkg/server/smtp/handler.go | 9 +- pkg/storage/file/fstore.go | 5 +- pkg/stringutil/utils.go | 208 ------------------------------ pkg/stringutil/utils_test.go | 220 +++----------------------------- pkg/webui/mailbox_controller.go | 18 +-- pkg/webui/root_controller.go | 4 +- 10 files changed, 388 insertions(+), 433 deletions(-) create mode 100644 pkg/policy/address.go create mode 100644 pkg/policy/address_test.go diff --git a/pkg/policy/address.go b/pkg/policy/address.go new file mode 100644 index 0000000..7cc85f0 --- /dev/null +++ b/pkg/policy/address.go @@ -0,0 +1,204 @@ +package policy + +import ( + "bytes" + "fmt" + "strings" +) + +// ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") +// and returns just the mailbox name (ex: "user"). Returns an error if +// localPart contains invalid characters; it won't accept any that must be +// quoted according to RFC3696. +func ParseMailboxName(localPart string) (result string, err error) { + if localPart == "" { + return "", fmt.Errorf("Mailbox name cannot be empty") + } + result = strings.ToLower(localPart) + invalid := make([]byte, 0, 10) + for i := 0; i < len(result); i++ { + c := result[i] + switch { + case 'a' <= c && c <= 'z': + case '0' <= c && c <= '9': + case bytes.IndexByte([]byte("!#$%&'*+-=/?^_`.{|}~"), c) >= 0: + default: + invalid = append(invalid, c) + } + } + if len(invalid) > 0 { + return "", fmt.Errorf("Mailbox name contained invalid character(s): %q", invalid) + } + if idx := strings.Index(result, "+"); idx > -1 { + result = result[0:idx] + } + return result, nil +} + +// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 +func ValidateDomainPart(domain string) bool { + if len(domain) == 0 { + return false + } + if len(domain) > 255 { + return false + } + if domain[len(domain)-1] != '.' { + domain += "." + } + prev := '.' + labelLen := 0 + hasAlphaNum := false + for _, c := range domain { + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9') || c == '_': + // Must contain some of these to be a valid label. + hasAlphaNum = true + labelLen++ + case c == '-': + if prev == '.' { + // Cannot lead with hyphen. + return false + } + case c == '.': + if prev == '.' || prev == '-' { + // Cannot end with hyphen or double-dot. + return false + } + if labelLen > 63 { + return false + } + if !hasAlphaNum { + return false + } + labelLen = 0 + hasAlphaNum = false + default: + // Unknown character. + return false + } + prev = c + } + return true +} + +// ParseEmailAddress unescapes an email address, and splits the local part from the domain part. +// An error is returned if the local or domain parts fail validation following the guidelines +// in RFC3696. +func ParseEmailAddress(address string) (local string, domain string, err error) { + if address == "" { + return "", "", fmt.Errorf("Empty address") + } + if len(address) > 320 { + return "", "", fmt.Errorf("Address exceeds 320 characters") + } + if address[0] == '@' { + return "", "", fmt.Errorf("Address cannot start with @ symbol") + } + if address[0] == '.' { + return "", "", fmt.Errorf("Address cannot start with a period") + } + // Loop over address parsing out local part. + buf := new(bytes.Buffer) + prev := byte('.') + inCharQuote := false + inStringQuote := false +LOOP: + for i := 0; i < len(address); i++ { + c := address[i] + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'): + // Letters are OK. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case '0' <= c && c <= '9': + // Numbers are OK. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case bytes.IndexByte([]byte("!#$%&'*+-/=?^_`{|}~"), c) >= 0: + // These specials can be used unquoted. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case c == '.': + // A single period is OK. + if prev == '.' { + // Sequence of periods is not permitted. + return "", "", fmt.Errorf("Sequence of periods is not permitted") + } + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case c == '\\': + inCharQuote = true + case c == '"': + if inCharQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else if inStringQuote { + inStringQuote = false + } else { + if i == 0 { + inStringQuote = true + } else { + return "", "", fmt.Errorf("Quoted string can only begin at start of address") + } + } + case c == '@': + if inCharQuote || inStringQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else { + // End of local-part. + if i > 128 { + return "", "", fmt.Errorf("Local part must not exceed 128 characters") + } + if prev == '.' { + return "", "", fmt.Errorf("Local part cannot end with a period") + } + domain = address[i+1:] + break LOOP + } + case c > 127: + return "", "", fmt.Errorf("Characters outside of US-ASCII range not permitted") + default: + if inCharQuote || inStringQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else { + return "", "", fmt.Errorf("Character %q must be quoted", c) + } + } + prev = c + } + if inCharQuote { + return "", "", fmt.Errorf("Cannot end address with unterminated quoted-pair") + } + if inStringQuote { + return "", "", fmt.Errorf("Cannot end address with unterminated string quote") + } + if !ValidateDomainPart(domain) { + return "", "", fmt.Errorf("Domain part validation failed") + } + return buf.String(), domain, nil +} diff --git a/pkg/policy/address_test.go b/pkg/policy/address_test.go new file mode 100644 index 0000000..149e9b3 --- /dev/null +++ b/pkg/policy/address_test.go @@ -0,0 +1,138 @@ +package policy_test + +import ( + "strings" + "testing" + + "github.com/jhillyerd/inbucket/pkg/policy" +) + +func TestParseMailboxName(t *testing.T) { + var validTable = []struct { + input string + expect string + }{ + {"mailbox", "mailbox"}, + {"user123", "user123"}, + {"MailBOX", "mailbox"}, + {"First.Last", "first.last"}, + {"user+label", "user"}, + {"chars!#$%", "chars!#$%"}, + {"chars&'*-", "chars&'*-"}, + {"chars=/?^", "chars=/?^"}, + {"chars_`.{", "chars_`.{"}, + {"chars|}~", "chars|}~"}, + } + for _, tt := range validTable { + if result, err := policy.ParseMailboxName(tt.input); err != nil { + t.Errorf("Error while parsing %q: %v", tt.input, err) + } else { + if result != tt.expect { + t.Errorf("Parsing %q, expected %q, got %q", tt.input, tt.expect, result) + } + } + } + var invalidTable = []struct { + input, msg string + }{ + {"", "Empty mailbox name is not permitted"}, + {"user@host", "@ symbol not permitted"}, + {"first last", "Space not permitted"}, + {"first\"last", "Double quote not permitted"}, + {"first\nlast", "Control chars not permitted"}, + } + for _, tt := range invalidTable { + if _, err := policy.ParseMailboxName(tt.input); err == nil { + t.Errorf("Didn't get an error while parsing %q: %v", tt.input, tt.msg) + } + } +} + +func TestValidateDomain(t *testing.T) { + var testTable = []struct { + input string + expect bool + msg string + }{ + {"", false, "Empty domain is not valid"}, + {"hostname", true, "Just a hostname is valid"}, + {"github.com", true, "Two labels should be just fine"}, + {"my-domain.com", true, "Hyphen is allowed mid-label"}, + {"_domainkey.foo.com", true, "Underscores are allowed"}, + {"bar.com.", true, "Must be able to end with a dot"}, + {"ABC.6DBS.com", true, "Mixed case is OK"}, + {"mail.123.com", true, "Number only label valid"}, + {"123.com", true, "Number only label valid"}, + {"google..com", false, "Double dot not valid"}, + {".foo.com", false, "Cannot start with a dot"}, + {"google\r.com", false, "Special chars not allowed"}, + {"foo.-bar.com", false, "Label cannot start with hyphen"}, + {"foo-.bar.com", false, "Label cannot end with hyphen"}, + {strings.Repeat("a", 256), false, "Max domain length is 255"}, + {strings.Repeat("a", 63) + ".com", true, "Should allow 63 char domain label"}, + {strings.Repeat("a", 64) + ".com", false, "Max domain label length is 63"}, + } + for _, tt := range testTable { + if policy.ValidateDomainPart(tt.input) != tt.expect { + t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) + } + } +} + +func TestValidateLocal(t *testing.T) { + var testTable = []struct { + input string + expect bool + msg string + }{ + {"", false, "Empty local is not valid"}, + {"a", true, "Single letter should be fine"}, + {strings.Repeat("a", 128), true, "Valid up to 128 characters"}, + {strings.Repeat("a", 129), false, "Only valid up to 128 characters"}, + {"FirstLast", true, "Mixed case permitted"}, + {"user123", true, "Numbers permitted"}, + {"a!#$%&'*+-/=?^_`{|}~", true, "Any of !#$%&'*+-/=?^_`{|}~ are permitted"}, + {"first.last", true, "Embedded period is permitted"}, + {"first..last", false, "Sequence of periods is not allowed"}, + {".user", false, "Cannot lead with a period"}, + {"user.", false, "Cannot end with a period"}, + {"james@mail", false, "Unquoted @ not permitted"}, + {"first last", false, "Unquoted space not permitted"}, + {"tricky\\. ", false, "Unquoted space not permitted"}, + {"no,commas", false, "Unquoted comma not allowed"}, + {"t[es]t", false, "Unquoted square brackets not allowed"}, + {"james\\", false, "Cannot end with backslash quote"}, + {"james\\@mail", true, "Quoted @ permitted"}, + {"quoted\\ space", true, "Quoted space permitted"}, + {"no\\,commas", true, "Quoted comma is OK"}, + {"t\\[es\\]t", true, "Quoted brackets are OK"}, + {"user\\name", true, "Should be able to quote a-z"}, + {"USER\\NAME", true, "Should be able to quote A-Z"}, + {"user\\1", true, "Should be able to quote a digit"}, + {"one\\$\\|", true, "Should be able to quote plain specials"}, + {"return\\\r", true, "Should be able to quote ASCII control chars"}, + {"high\\\x80", false, "Should not accept > 7-bit quoted chars"}, + {"quote\\\"", true, "Quoted double quote is permitted"}, + {"\"james\"", true, "Quoted a-z is permitted"}, + {"\"first last\"", true, "Quoted space is permitted"}, + {"\"quoted@sign\"", true, "Quoted @ is allowed"}, + {"\"qp\\\"quote\"", true, "Quoted quote within quoted string is OK"}, + {"\"unterminated", false, "Quoted string must be terminated"}, + {"\"unterminated\\\"", false, "Quoted string must be terminated"}, + {"embed\"quote\"string", false, "Embedded quoted string is illegal"}, + {"user+mailbox", true, "RFC3696 test case should be valid"}, + {"customer/department=shipping", true, "RFC3696 test case should be valid"}, + {"$A12345", true, "RFC3696 test case should be valid"}, + {"!def!xyz%abc", true, "RFC3696 test case should be valid"}, + {"_somename", true, "RFC3696 test case should be valid"}, + } + for _, tt := range testTable { + _, _, err := policy.ParseEmailAddress(tt.input + "@domain.com") + if (err != nil) == tt.expect { + if err != nil { + t.Logf("Got error: %s", err) + } + t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) + } + } +} diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 7b40201..2b10eb0 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -11,6 +11,7 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -20,7 +21,7 @@ import ( // MailboxListV1 renders a list of messages in a mailbox func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -50,7 +51,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -100,7 +101,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // MailboxPurgeV1 deletes all messages from a mailbox func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -118,7 +119,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -142,7 +143,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index 5ad2dbf..7614ac1 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -7,9 +7,9 @@ import ( "github.com/gorilla/websocket" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) const ( @@ -173,7 +173,7 @@ func MonitorAllMessagesV1( // notifies the client of messages received by a particular mailbox. func MonitorMailboxMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index d9dad39..e123e9f 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -267,7 +268,7 @@ func (ss *Session) readyHandler(cmd string, arg string) { return } from := m[1] - if _, _, err := stringutil.ParseEmailAddress(from); err != nil { + if _, _, err := policy.ParseEmailAddress(from); err != nil { ss.send("501 Bad sender address syntax") ss.logWarn("Bad address as MAIL arg: %q, %s", from, err) return @@ -316,7 +317,7 @@ func (ss *Session) mailHandler(cmd string, arg string) { } // This trim is probably too forgiving recip := strings.Trim(arg[3:], "<> ") - if _, _, err := stringutil.ParseEmailAddress(recip); err != nil { + if _, _, err := policy.ParseEmailAddress(recip); err != nil { ss.send("501 Bad recipient address syntax") ss.logWarn("Bad address as RCPT arg: %q, %s", recip, err) return @@ -355,7 +356,7 @@ func (ss *Session) dataHandler() { if ss.server.storeMessages { for e := ss.recipients.Front(); e != nil; e = e.Next() { recip := e.Value.(string) - local, domain, err := stringutil.ParseEmailAddress(recip) + local, domain, err := policy.ParseEmailAddress(recip) if err != nil { ss.logError("Failed to parse address for %q", recip) ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", recip)) @@ -436,7 +437,7 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { - name, err := stringutil.ParseMailboxName(r.localPart) + name, err := policy.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index b41b5d3..dc20b5d 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -13,6 +13,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -218,7 +219,7 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro // LockFor returns the RWMutex for this mailbox, or an error. func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { - name, err := stringutil.ParseMailboxName(emailAddress) + name, err := policy.ParseMailboxName(emailAddress) if err != nil { return nil, err } @@ -237,7 +238,7 @@ func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { // mbox returns the named mailbox. func (fs *Store) mbox(mailbox string) (*mbox, error) { - name, err := stringutil.ParseMailboxName(mailbox) + name, err := policy.ParseMailboxName(mailbox) if err != nil { return nil, err } diff --git a/pkg/stringutil/utils.go b/pkg/stringutil/utils.go index 193e552..699eff8 100644 --- a/pkg/stringutil/utils.go +++ b/pkg/stringutil/utils.go @@ -1,47 +1,12 @@ package stringutil import ( - "bytes" "crypto/sha1" "fmt" "io" "net/mail" - "strings" ) -// ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") -// and returns just the mailbox name (ex: "user"). Returns an error if -// localPart contains invalid characters; it won't accept any that must be -// quoted according to RFC3696. -func ParseMailboxName(localPart string) (result string, err error) { - if localPart == "" { - return "", fmt.Errorf("Mailbox name cannot be empty") - } - result = strings.ToLower(localPart) - - invalid := make([]byte, 0, 10) - - for i := 0; i < len(result); i++ { - c := result[i] - switch { - case 'a' <= c && c <= 'z': - case '0' <= c && c <= '9': - case bytes.IndexByte([]byte("!#$%&'*+-=/?^_`.{|}~"), c) >= 0: - default: - invalid = append(invalid, c) - } - } - - if len(invalid) > 0 { - return "", fmt.Errorf("Mailbox name contained invalid character(s): %q", invalid) - } - - if idx := strings.Index(result, "+"); idx > -1 { - result = result[0:idx] - } - return result, nil -} - // HashMailboxName accepts a mailbox name and hashes it. filestore uses this as // the directory to house the mailbox func HashMailboxName(mailbox string) string { @@ -53,179 +18,6 @@ func HashMailboxName(mailbox string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 -func ValidateDomainPart(domain string) bool { - if len(domain) == 0 { - return false - } - if len(domain) > 255 { - return false - } - if domain[len(domain)-1] != '.' { - domain += "." - } - prev := '.' - labelLen := 0 - hasAlphaNum := false - - for _, c := range domain { - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9') || c == '_': - // Must contain some of these to be a valid label - hasAlphaNum = true - labelLen++ - case c == '-': - if prev == '.' { - // Cannot lead with hyphen - return false - } - case c == '.': - if prev == '.' || prev == '-' { - // Cannot end with hyphen or double-dot - return false - } - if labelLen > 63 { - return false - } - if !hasAlphaNum { - return false - } - labelLen = 0 - hasAlphaNum = false - default: - // Unknown character - return false - } - prev = c - } - - return true -} - -// ParseEmailAddress unescapes an email address, and splits the local part from the domain part. -// An error is returned if the local or domain parts fail validation following the guidelines -// in RFC3696. -func ParseEmailAddress(address string) (local string, domain string, err error) { - if address == "" { - return "", "", fmt.Errorf("Empty address") - } - if len(address) > 320 { - return "", "", fmt.Errorf("Address exceeds 320 characters") - } - if address[0] == '@' { - return "", "", fmt.Errorf("Address cannot start with @ symbol") - } - if address[0] == '.' { - return "", "", fmt.Errorf("Address cannot start with a period") - } - - // Loop over address parsing out local part - buf := new(bytes.Buffer) - prev := byte('.') - inCharQuote := false - inStringQuote := false -LOOP: - for i := 0; i < len(address); i++ { - c := address[i] - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'): - // Letters are OK - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case '0' <= c && c <= '9': - // Numbers are OK - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case bytes.IndexByte([]byte("!#$%&'*+-/=?^_`{|}~"), c) >= 0: - // These specials can be used unquoted - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case c == '.': - // A single period is OK - if prev == '.' { - // Sequence of periods is not permitted - return "", "", fmt.Errorf("Sequence of periods is not permitted") - } - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case c == '\\': - inCharQuote = true - case c == '"': - if inCharQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else if inStringQuote { - inStringQuote = false - } else { - if i == 0 { - inStringQuote = true - } else { - return "", "", fmt.Errorf("Quoted string can only begin at start of address") - } - } - case c == '@': - if inCharQuote || inStringQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else { - // End of local-part - if i > 128 { - return "", "", fmt.Errorf("Local part must not exceed 128 characters") - } - if prev == '.' { - return "", "", fmt.Errorf("Local part cannot end with a period") - } - domain = address[i+1:] - break LOOP - } - case c > 127: - return "", "", fmt.Errorf("Characters outside of US-ASCII range not permitted") - default: - if inCharQuote || inStringQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else { - return "", "", fmt.Errorf("Character %q must be quoted", c) - } - } - prev = c - } - if inCharQuote { - return "", "", fmt.Errorf("Cannot end address with unterminated quoted-pair") - } - if inStringQuote { - return "", "", fmt.Errorf("Cannot end address with unterminated string quote") - } - - if !ValidateDomainPart(domain) { - return "", "", fmt.Errorf("Domain part validation failed") - } - - return buf.String(), domain, nil -} - // StringAddressList converts a list of addresses to a list of strings func StringAddressList(addrs []*mail.Address) []string { s := make([]string, len(addrs)) diff --git a/pkg/stringutil/utils_test.go b/pkg/stringutil/utils_test.go index 330bfbd..8c0b7bd 100644 --- a/pkg/stringutil/utils_test.go +++ b/pkg/stringutil/utils_test.go @@ -1,215 +1,33 @@ -package stringutil +package stringutil_test import ( - "strings" + "net/mail" "testing" - "github.com/stretchr/testify/assert" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) -func TestParseMailboxName(t *testing.T) { - var validTable = []struct { - input string - expect string - }{ - {"mailbox", "mailbox"}, - {"user123", "user123"}, - {"MailBOX", "mailbox"}, - {"First.Last", "first.last"}, - {"user+label", "user"}, - {"chars!#$%", "chars!#$%"}, - {"chars&'*-", "chars&'*-"}, - {"chars=/?^", "chars=/?^"}, - {"chars_`.{", "chars_`.{"}, - {"chars|}~", "chars|}~"}, - } - - for _, tt := range validTable { - if result, err := ParseMailboxName(tt.input); err != nil { - t.Errorf("Error while parsing %q: %v", tt.input, err) - } else { - if result != tt.expect { - t.Errorf("Parsing %q, expected %q, got %q", tt.input, tt.expect, result) - } - } - } - - var invalidTable = []struct { - input, msg string - }{ - {"", "Empty mailbox name is not permitted"}, - {"user@host", "@ symbol not permitted"}, - {"first last", "Space not permitted"}, - {"first\"last", "Double quote not permitted"}, - {"first\nlast", "Control chars not permitted"}, - } - - for _, tt := range invalidTable { - if _, err := ParseMailboxName(tt.input); err == nil { - t.Errorf("Didn't get an error while parsing %q: %v", tt.input, tt.msg) - } - } -} - func TestHashMailboxName(t *testing.T) { - assert.Equal(t, HashMailboxName("mail"), "1d6e1cf70ec6f9ab28d3ea4b27a49a77654d370e") -} - -func TestValidateDomain(t *testing.T) { - assert.False(t, ValidateDomainPart(strings.Repeat("a", 256)), - "Max domain length is 255") - assert.False(t, ValidateDomainPart(strings.Repeat("a", 64)+".com"), - "Max label length is 63") - assert.True(t, ValidateDomainPart(strings.Repeat("a", 63)+".com"), - "Should allow 63 char label") - - var testTable = []struct { - input string - expect bool - msg string - }{ - {"", false, "Empty domain is not valid"}, - {"hostname", true, "Just a hostname is valid"}, - {"github.com", true, "Two labels should be just fine"}, - {"my-domain.com", true, "Hyphen is allowed mid-label"}, - {"_domainkey.foo.com", true, "Underscores are allowed"}, - {"bar.com.", true, "Must be able to end with a dot"}, - {"ABC.6DBS.com", true, "Mixed case is OK"}, - {"mail.123.com", true, "Number only label valid"}, - {"123.com", true, "Number only label valid"}, - {"google..com", false, "Double dot not valid"}, - {".foo.com", false, "Cannot start with a dot"}, - {"google\r.com", false, "Special chars not allowed"}, - {"foo.-bar.com", false, "Label cannot start with hyphen"}, - {"foo-.bar.com", false, "Label cannot end with hyphen"}, - } - - for _, tt := range testTable { - if ValidateDomainPart(tt.input) != tt.expect { - t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) - } + want := "1d6e1cf70ec6f9ab28d3ea4b27a49a77654d370e" + got := stringutil.HashMailboxName("mail") + if got != want { + t.Errorf("Got %q, want %q", got, want) } } -func TestValidateLocal(t *testing.T) { - var testTable = []struct { - input string - expect bool - msg string - }{ - {"", false, "Empty local is not valid"}, - {"a", true, "Single letter should be fine"}, - {strings.Repeat("a", 128), true, "Valid up to 128 characters"}, - {strings.Repeat("a", 129), false, "Only valid up to 128 characters"}, - {"FirstLast", true, "Mixed case permitted"}, - {"user123", true, "Numbers permitted"}, - {"a!#$%&'*+-/=?^_`{|}~", true, "Any of !#$%&'*+-/=?^_`{|}~ are permitted"}, - {"first.last", true, "Embedded period is permitted"}, - {"first..last", false, "Sequence of periods is not allowed"}, - {".user", false, "Cannot lead with a period"}, - {"user.", false, "Cannot end with a period"}, - {"james@mail", false, "Unquoted @ not permitted"}, - {"first last", false, "Unquoted space not permitted"}, - {"tricky\\. ", false, "Unquoted space not permitted"}, - {"no,commas", false, "Unquoted comma not allowed"}, - {"t[es]t", false, "Unquoted square brackets not allowed"}, - {"james\\", false, "Cannot end with backslash quote"}, - {"james\\@mail", true, "Quoted @ permitted"}, - {"quoted\\ space", true, "Quoted space permitted"}, - {"no\\,commas", true, "Quoted comma is OK"}, - {"t\\[es\\]t", true, "Quoted brackets are OK"}, - {"user\\name", true, "Should be able to quote a-z"}, - {"USER\\NAME", true, "Should be able to quote A-Z"}, - {"user\\1", true, "Should be able to quote a digit"}, - {"one\\$\\|", true, "Should be able to quote plain specials"}, - {"return\\\r", true, "Should be able to quote ASCII control chars"}, - {"high\\\x80", false, "Should not accept > 7-bit quoted chars"}, - {"quote\\\"", true, "Quoted double quote is permitted"}, - {"\"james\"", true, "Quoted a-z is permitted"}, - {"\"first last\"", true, "Quoted space is permitted"}, - {"\"quoted@sign\"", true, "Quoted @ is allowed"}, - {"\"qp\\\"quote\"", true, "Quoted quote within quoted string is OK"}, - {"\"unterminated", false, "Quoted string must be terminated"}, - {"\"unterminated\\\"", false, "Quoted string must be terminated"}, - {"embed\"quote\"string", false, "Embedded quoted string is illegal"}, - - {"user+mailbox", true, "RFC3696 test case should be valid"}, - {"customer/department=shipping", true, "RFC3696 test case should be valid"}, - {"$A12345", true, "RFC3696 test case should be valid"}, - {"!def!xyz%abc", true, "RFC3696 test case should be valid"}, - {"_somename", true, "RFC3696 test case should be valid"}, +func TestStringAddressList(t *testing.T) { + input := []*mail.Address{ + {Name: "Fred B. Fish", Address: "fred@fish.org"}, + {Name: "User", Address: "user@domain.org"}, } - - for _, tt := range testTable { - _, _, err := ParseEmailAddress(tt.input + "@domain.com") - if (err != nil) == tt.expect { - if err != nil { - t.Logf("Got error: %s", err) - } - t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) - } - } -} - -func TestParseEmailAddress(t *testing.T) { - // Test some good email addresses - var testTable = []struct { - input, local, domain string - }{ - {"root@localhost", "root", "localhost"}, - {"FirstLast@domain.local", "FirstLast", "domain.local"}, - {"route66@prodigy.net", "route66", "prodigy.net"}, - {"lorbit!user@uucp", "lorbit!user", "uucp"}, - {"user+spam@gmail.com", "user+spam", "gmail.com"}, - {"first.last@domain.local", "first.last", "domain.local"}, - {"first\\ last@_key.domain.com", "first last", "_key.domain.com"}, - {"first\\\"last@a.b.c", "first\"last", "a.b.c"}, - {"user\\@internal@myhost.ca", "user@internal", "myhost.ca"}, - {"\"first last@evil\"@top-secret.gov", "first last@evil", "top-secret.gov"}, - {"\"line\nfeed\"@linenoise.co.uk", "line\nfeed", "linenoise.co.uk"}, - {"user+mailbox@host", "user+mailbox", "host"}, - {"customer/department=shipping@host", "customer/department=shipping", "host"}, - {"$A12345@host", "$A12345", "host"}, - {"!def!xyz%abc@host", "!def!xyz%abc", "host"}, - {"_somename@host", "_somename", "host"}, - } - - for _, tt := range testTable { - local, domain, err := ParseEmailAddress(tt.input) - if err != nil { - t.Errorf("Error when parsing %q: %s", tt.input, err) - } else { - if tt.local != local { - t.Errorf("When parsing %q, expected local %q, got %q instead", - tt.input, tt.local, local) - } - if tt.domain != domain { - t.Errorf("When parsing %q, expected domain %q, got %q instead", - tt.input, tt.domain, domain) - } - } - } - - // Check that validations fail correctly - var badTable = []struct { - input, msg string - }{ - {"", "Empty address not permitted"}, - {"user", "Missing domain part"}, - {"@host", "Missing local part"}, - {"user\\@host", "Missing domain part"}, - {"\"user@host\"", "Missing domain part"}, - {"\"user@host", "Unterminated quoted string"}, - {"first last@host", "Unquoted space"}, - {"user@bad!domain", "Invalid domain"}, - {".user@host", "Can't lead with a ."}, - {"user.@host", "Can't end local with a dot"}, - {"user@bad domain", "No spaces in domain permitted"}, - } - - for _, tt := range badTable { - if _, _, err := ParseEmailAddress(tt.input); err == nil { - t.Errorf("Did not get expected error when parsing %q: %s", tt.input, tt.msg) + want := []string{`"Fred B. Fish" `, `"User" `} + output := stringutil.StringAddressList(input) + if len(output) != len(want) { + t.Fatalf("Got %v strings, want: %v", len(output), len(want)) + } + for i, got := range output { + if got != want[i] { + t.Errorf("Got %q, want: %q", got, want[i]) } } } diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 16aa305..2af9ef1 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" - "github.com/jhillyerd/inbucket/pkg/stringutil" "github.com/jhillyerd/inbucket/pkg/webui/sanitize" ) @@ -25,7 +25,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err = stringutil.ParseMailboxName(name) + name, err = policy.ParseMailboxName(name) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -52,7 +52,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -68,7 +68,7 @@ func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // MailboxList renders a list of messages in a mailbox. Renders a partial func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -90,7 +90,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -131,7 +131,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -159,7 +159,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -183,7 +183,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -225,7 +225,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // MailboxViewAttach sends the attachment to the client for online viewing func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) diff --git a/pkg/webui/root_controller.go b/pkg/webui/root_controller.go index 85d1662..21d4dd3 100644 --- a/pkg/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -7,8 +7,8 @@ import ( "net/http" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) // RootIndex serves the Inbucket landing page @@ -58,7 +58,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Conte http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) From 469a778d81923cc221cc6dc3fd4af65fe97ba978 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 11:15:17 -0700 Subject: [PATCH 20/82] policy: Impl Addressing{} and Recipient{} for #84 --- Makefile | 2 +- pkg/policy/address.go | 136 ++++++++++++++++++++++++------------- pkg/policy/address_test.go | 53 +++++++++++++++ pkg/policy/recipient.go | 25 +++++++ 4 files changed, 167 insertions(+), 49 deletions(-) create mode 100644 pkg/policy/recipient.go diff --git a/Makefile b/Makefile index eacf4ba..ff275c0 100644 --- a/Makefile +++ b/Makefile @@ -36,4 +36,4 @@ lint: @go vet $(PKGS) reflex: - reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./...' + reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./... && echo ALL PASS' diff --git a/pkg/policy/address.go b/pkg/policy/address.go index 7cc85f0..34ed8f8 100644 --- a/pkg/policy/address.go +++ b/pkg/policy/address.go @@ -3,9 +3,48 @@ package policy import ( "bytes" "fmt" + "net/mail" "strings" + + "github.com/jhillyerd/inbucket/pkg/config" ) +// Addressing handles email address policy. +type Addressing struct { + Config config.SMTPConfig +} + +// NewRecipient parses an address into a Recipient. +func (a *Addressing) NewRecipient(address string) (*Recipient, error) { + local, domain, err := ParseEmailAddress(address) + if err != nil { + return nil, err + } + mailbox, err := ParseMailboxName(local) + if err != nil { + return nil, err + } + ar, err := mail.ParseAddress(address) + if err != nil { + return nil, err + } + return &Recipient{ + Address: *ar, + apolicy: a, + LocalPart: local, + Domain: domain, + Mailbox: mailbox, + }, nil +} + +// ShouldStoreDomain indicates if Inbucket stores email destined for the specified domain. +func (a *Addressing) ShouldStoreDomain(domain string) bool { + if a.Config.StoreMessages { + return strings.ToLower(domain) != strings.ToLower(a.Config.DomainNoStore) + } + return false +} + // ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") // and returns just the mailbox name (ex: "user"). Returns an error if // localPart contains invalid characters; it won't accept any that must be @@ -35,54 +74,6 @@ func ParseMailboxName(localPart string) (result string, err error) { return result, nil } -// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 -func ValidateDomainPart(domain string) bool { - if len(domain) == 0 { - return false - } - if len(domain) > 255 { - return false - } - if domain[len(domain)-1] != '.' { - domain += "." - } - prev := '.' - labelLen := 0 - hasAlphaNum := false - for _, c := range domain { - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9') || c == '_': - // Must contain some of these to be a valid label. - hasAlphaNum = true - labelLen++ - case c == '-': - if prev == '.' { - // Cannot lead with hyphen. - return false - } - case c == '.': - if prev == '.' || prev == '-' { - // Cannot end with hyphen or double-dot. - return false - } - if labelLen > 63 { - return false - } - if !hasAlphaNum { - return false - } - labelLen = 0 - hasAlphaNum = false - default: - // Unknown character. - return false - } - prev = c - } - return true -} - // ParseEmailAddress unescapes an email address, and splits the local part from the domain part. // An error is returned if the local or domain parts fail validation following the guidelines // in RFC3696. @@ -202,3 +193,52 @@ LOOP: } return buf.String(), domain, nil } + +// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035. Used by +// ParseEmailAddress(). +func ValidateDomainPart(domain string) bool { + if len(domain) == 0 { + return false + } + if len(domain) > 255 { + return false + } + if domain[len(domain)-1] != '.' { + domain += "." + } + prev := '.' + labelLen := 0 + hasAlphaNum := false + for _, c := range domain { + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9') || c == '_': + // Must contain some of these to be a valid label. + hasAlphaNum = true + labelLen++ + case c == '-': + if prev == '.' { + // Cannot lead with hyphen. + return false + } + case c == '.': + if prev == '.' || prev == '-' { + // Cannot end with hyphen or double-dot. + return false + } + if labelLen > 63 { + return false + } + if !hasAlphaNum { + return false + } + labelLen = 0 + hasAlphaNum = false + default: + // Unknown character. + return false + } + prev = c + } + return true +} diff --git a/pkg/policy/address_test.go b/pkg/policy/address_test.go index 149e9b3..2ab2c60 100644 --- a/pkg/policy/address_test.go +++ b/pkg/policy/address_test.go @@ -4,9 +4,62 @@ import ( "strings" "testing" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/policy" ) +func TestShouldStoreDomain(t *testing.T) { + // Test with storage enabled. + ap := &policy.Addressing{ + Config: config.SMTPConfig{ + DomainNoStore: "Foo.Com", + StoreMessages: true, + }, + } + testCases := []struct { + domain string + want bool + }{ + {domain: "bar.com", want: true}, + {domain: "foo.com", want: false}, + {domain: "FOO.com", want: false}, + {domain: "bar.foo.com", want: true}, + } + for _, tc := range testCases { + t.Run(tc.domain, func(t *testing.T) { + got := ap.ShouldStoreDomain(tc.domain) + if got != tc.want { + t.Errorf("Got %v for %q, want: %v", got, tc.domain, tc.want) + } + + }) + } + // Test with storage disabled. + ap = &policy.Addressing{ + Config: config.SMTPConfig{ + StoreMessages: false, + }, + } + testCases = []struct { + domain string + want bool + }{ + {domain: "bar.com", want: false}, + {domain: "foo.com", want: false}, + {domain: "FOO.com", want: false}, + {domain: "bar.foo.com", want: false}, + } + for _, tc := range testCases { + t.Run(tc.domain, func(t *testing.T) { + got := ap.ShouldStoreDomain(tc.domain) + if got != tc.want { + t.Errorf("Got %v for %q, want: %v", got, tc.domain, tc.want) + } + + }) + } +} + func TestParseMailboxName(t *testing.T) { var validTable = []struct { input string diff --git a/pkg/policy/recipient.go b/pkg/policy/recipient.go new file mode 100644 index 0000000..36fd94b --- /dev/null +++ b/pkg/policy/recipient.go @@ -0,0 +1,25 @@ +package policy + +import "net/mail" + +// Recipient represents a potential email recipient, allows policies for it to be queried. +type Recipient struct { + mail.Address + apolicy *Addressing + // LocalPart is the part of the address before @, including +extension. + LocalPart string + // Domain is the part of the address after @. + Domain string + // Mailbox is the canonical mailbox name for this recipient. + Mailbox string +} + +// ShouldAccept returns true if Inbucket should accept mail for this recipient. +func (r *Recipient) ShouldAccept() bool { + return true +} + +// ShouldStore returns true if Inbucket should store mail for this recipient. +func (r *Recipient) ShouldStore() bool { + return r.apolicy.ShouldStoreDomain(r.Domain) +} From b9003a9328ac7a939b2e044c55b9101d550f0dfb Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 12:39:09 -0700 Subject: [PATCH 21/82] smtp: Wire in policy.Recipient for #84 --- cmd/inbucket/main.go | 4 +- pkg/server/smtp/handler.go | 90 ++++++++++++--------------------- pkg/server/smtp/handler_test.go | 9 ++-- pkg/server/smtp/listener.go | 10 ++-- 4 files changed, 47 insertions(+), 66 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 7020676..ff5a313 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest" "github.com/jhillyerd/inbucket/pkg/server/pop3" "github.com/jhillyerd/inbucket/pkg/server/smtp" @@ -135,7 +136,8 @@ func main() { go pop3Server.Start(rootCtx) // Startup SMTP server - smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, msgHub) + apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} + smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, apolicy, msgHub) go smtpServer.Start(rootCtx) // Loop forever waiting for signals or shutdown channel diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index e123e9f..f15656b 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -3,7 +3,6 @@ package smtp import ( "bufio" "bytes" - "container/list" "fmt" "io" "net" @@ -72,11 +71,6 @@ var commands = map[string]bool{ "TURN": true, } -// recipientDetails for message delivery -type recipientDetails struct { - address, localPart, domainPart string -} - // Session holds the state of an SMTP session type Session struct { server *Server @@ -88,14 +82,22 @@ type Session struct { state State reader *bufio.Reader from string - recipients *list.List + recipients []*policy.Recipient } // NewSession creates a new Session for the given connection func NewSession(server *Server, id int, conn net.Conn) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) - return &Session{server: server, id: id, conn: conn, state: GREET, reader: reader, remoteHost: host} + return &Session{ + server: server, + id: id, + conn: conn, + state: GREET, + reader: reader, + remoteHost: host, + recipients: make([]*policy.Recipient, 0), + } } func (ss *Session) String() string { @@ -297,7 +299,6 @@ func (ss *Session) readyHandler(cmd string, arg string) { } } ss.from = from - ss.recipients = list.New() ss.logInfo("Mail from: %v", from) ss.send(fmt.Sprintf("250 Roger, accepting mail from <%v>", from)) ss.enterState(MAIL) @@ -316,20 +317,21 @@ func (ss *Session) mailHandler(cmd string, arg string) { return } // This trim is probably too forgiving - recip := strings.Trim(arg[3:], "<> ") - if _, _, err := policy.ParseEmailAddress(recip); err != nil { + addr := strings.Trim(arg[3:], "<> ") + recip, err := ss.server.apolicy.NewRecipient(addr) + if err != nil { ss.send("501 Bad recipient address syntax") - ss.logWarn("Bad address as RCPT arg: %q, %s", recip, err) + ss.logWarn("Bad address as RCPT arg: %q, %s", addr, err) return } - if ss.recipients.Len() >= ss.server.maxRecips { + if len(ss.recipients) >= ss.server.maxRecips { ss.logWarn("Maximum limit of %v recipients reached", ss.server.maxRecips) ss.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", ss.server.maxRecips)) return } - ss.recipients.PushBack(recip) - ss.logInfo("Recipient: %v", recip) - ss.send(fmt.Sprintf("250 I'll make sure <%v> gets this", recip)) + ss.recipients = append(ss.recipients, recip) + ss.logInfo("Recipient: %v", addr) + ss.send(fmt.Sprintf("250 I'll make sure <%v> gets this", addr)) return case "DATA": if arg != "" { @@ -337,7 +339,7 @@ func (ss *Session) mailHandler(cmd string, arg string) { ss.logWarn("Got unexpected args on DATA: %q", arg) return } - if ss.recipients.Len() > 0 { + if len(ss.recipients) > 0 { // We have recipients, go to accept data ss.enterState(DATA) return @@ -351,27 +353,6 @@ func (ss *Session) mailHandler(cmd string, arg string) { // DATA func (ss *Session) dataHandler() { - recipients := make([]recipientDetails, 0, ss.recipients.Len()) - // Get a Mailbox and a new Message for each recipient - if ss.server.storeMessages { - for e := ss.recipients.Front(); e != nil; e = e.Next() { - recip := e.Value.(string) - local, domain, err := policy.ParseEmailAddress(recip) - if err != nil { - ss.logError("Failed to parse address for %q", recip) - ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", recip)) - ss.reset() - return - } - if strings.ToLower(domain) != ss.server.domainNoStore { - // Not our "no store" domain, so store the message - recipients = append(recipients, recipientDetails{recip, local, domain}) - } else { - log.Tracef("Not storing message for %q", recip) - } - } - } - ss.send("354 Start mail input; end with .") msgBuf := &bytes.Buffer{} for { @@ -388,31 +369,27 @@ func (ss *Session) dataHandler() { } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete - if ss.server.storeMessages { - // Create a message for each valid recipient - for _, r := range recipients { + for _, recip := range ss.recipients { + if recip.ShouldStore() { // TODO temporary hack to fix #77 until datastore revamp - mu, err := ss.server.dataStore.LockFor(r.localPart) + mu, err := ss.server.dataStore.LockFor(recip.LocalPart) if err != nil { - ss.logError("Failed to get lock for %q: %s", r.localPart, err) + ss.logError("Failed to get lock for %q: %s", recip.LocalPart, err) // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", r.localPart)) + ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return } mu.Lock() - ok := ss.deliverMessage(r, msgBuf.Bytes()) + ok := ss.deliverMessage(recip, msgBuf.Bytes()) mu.Unlock() - if ok { - expReceivedTotal.Add(1) - } else { + if !ok { // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", r.localPart)) + ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return } } - } else { expReceivedTotal.Add(1) } ss.send("250 Mail accepted for delivery") @@ -436,8 +413,8 @@ func (ss *Session) dataHandler() { } // deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { - name, err := policy.ParseMailboxName(r.localPart) +func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { + name, err := policy.ParseMailboxName(recip.LocalPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false @@ -445,7 +422,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) // TODO replace with something that only reads header? env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { - ss.logError("Failed to parse message for %q: %v", r.localPart, err) + ss.logError("Failed to parse message for %q: %v", recip.LocalPart, err) return false } from, err := env.AddressList("From") @@ -461,7 +438,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) // Generate Received header. stamp := time.Now().Format(timeStampFormat) recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp)) + ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ Mailbox: name, @@ -474,7 +451,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) } id, err := ss.server.dataStore.AddMessage(delivery) if err != nil { - ss.logError("Failed to store message for %q: %s", r.localPart, err) + ss.logError("Failed to store message for %q: %s", recip.LocalPart, err) return false } // Broadcast message information. @@ -519,8 +496,7 @@ func (ss *Session) send(msg string) { ss.logTrace(">> %v >>", msg) } -// readByteLine reads a line of input into the provided buffer. Does -// not reset the Buffer - please do so prior to calling. +// readByteLine reads a line of input, returns byte slice. func (ss *Session) readByteLine() ([]byte, error) { if err := ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { return nil, err diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 94b0d48..6242b20 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -15,6 +15,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -172,10 +173,7 @@ func TestMailState(t *testing.T) { {"RCPT TO: u4@gmail.com", 250}, {"RSET", 250}, {"MAIL FROM:", 250}, - {"RCPT TO:name@host.com>", 250}, - {"RCPT TO:<\"user>name\"@host.com>", 250}, + {`RCPT TO:<"first/last"@host.com`, 250}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -360,7 +358,8 @@ func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown f close(shutdownChan) cancel() } - s = NewServer(cfg, shutdownChan, ds, msghub.New(ctx, 100)) + apolicy := &policy.Addressing{Config: cfg} + s = NewServer(cfg, shutdownChan, ds, apolicy, msghub.New(ctx, 100)) return s, buf, teardown } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 83d5697..0e795b7 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -13,6 +13,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -48,9 +49,10 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info + dataStore storage.Store // Mailbox/message store + apolicy *policy.Addressing // Address policy + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info // State listener net.Listener // Incoming network connections @@ -83,6 +85,7 @@ func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, ds storage.Store, + apolicy *policy.Addressing, msgHub *msghub.Hub) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), @@ -94,6 +97,7 @@ func NewServer( storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, dataStore: ds, + apolicy: apolicy, msgHub: msgHub, waitgroup: new(sync.WaitGroup), } From e84b1f89520a92d3f906143a2c440cda4629c621 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 14:02:50 -0700 Subject: [PATCH 22/82] storage: Make locking an implementation detail for #69 - file: Store handles its own locking #77 - file: Move mbox into its own file - file & test: remove LockFor() --- pkg/server/smtp/handler.go | 32 +--- pkg/storage/file/fstore.go | 296 +++++-------------------------------- pkg/storage/file/mbox.go | 242 ++++++++++++++++++++++++++++++ pkg/storage/storage.go | 3 - pkg/test/storage.go | 7 - 5 files changed, 288 insertions(+), 292 deletions(-) create mode 100644 pkg/storage/file/mbox.go diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index f15656b..1848fe2 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -368,23 +368,10 @@ func (ss *Session) dataHandler() { return } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { - // Mail data complete + // Mail data complete. for _, recip := range ss.recipients { if recip.ShouldStore() { - // TODO temporary hack to fix #77 until datastore revamp - mu, err := ss.server.dataStore.LockFor(recip.LocalPart) - if err != nil { - ss.logError("Failed to get lock for %q: %s", recip.LocalPart, err) - // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) - ss.reset() - return - } - mu.Lock() - ok := ss.deliverMessage(recip, msgBuf.Bytes()) - mu.Unlock() - if !ok { - // Delivery failure + if ok := ss.deliverMessage(recip, msgBuf.Bytes()); !ok { ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return @@ -397,28 +384,22 @@ func (ss *Session) dataHandler() { ss.reset() return } - // SMTP RFC says remove leading periods from input + // RFC says remove leading periods from input. if len(lineBuf) > 0 && lineBuf[0] == '.' { lineBuf = lineBuf[1:] } msgBuf.Write(lineBuf) if msgBuf.Len() > ss.server.maxMessageBytes { - // Max message size exceeded ss.send("552 Maximum message size exceeded") ss.logWarn("Max message size exceeded while in DATA") ss.reset() return } - } // end for + } } // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { - name, err := policy.ParseMailboxName(recip.LocalPart) - if err != nil { - // This parse already succeeded when MailboxFor was called, shouldn't fail here. - return false - } // TODO replace with something that only reads header? env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { @@ -441,7 +422,7 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ - Mailbox: name, + Mailbox: recip.Mailbox, From: from[0], To: to, Date: time.Now(), @@ -455,8 +436,9 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b return false } // Broadcast message information. + // TODO this belongs in message pkg. broadcast := msghub.Message{ - Mailbox: name, + Mailbox: recip.Mailbox, ID: id, From: delivery.From().String(), To: stringutil.StringAddressList(delivery.To()), diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index dc20b5d..f6a60ec 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -2,7 +2,6 @@ package file import ( "bufio" - "encoding/gob" "fmt" "io" "io/ioutil" @@ -77,11 +76,13 @@ func New(cfg config.DataStoreConfig) storage.Store { // AddMessage adds a message to the specified mailbox. func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { - r, err := m.RawReader() + mb, err := fs.mbox(m.Mailbox()) if err != nil { return "", err } - mb, err := fs.mbox(m.Mailbox()) + mb.Lock() + defer mb.Unlock() + r, err := m.RawReader() if err != nil { return "", err } @@ -140,6 +141,8 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { if err != nil { return nil, err } + mb.RLock() + defer mb.RUnlock() return mb.getMessage(id) } @@ -149,6 +152,8 @@ func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { if err != nil { return nil, err } + mb.RLock() + defer mb.RUnlock() return mb.getMessages() } @@ -158,6 +163,8 @@ func (fs *Store) RemoveMessage(mailbox, id string) error { if err != nil { return err } + mb.Lock() + defer mb.Unlock() return mb.removeMessage(id) } @@ -167,6 +174,8 @@ func (fs *Store) PurgeMessages(mailbox string) error { if err != nil { return err } + mb.Lock() + defer mb.Unlock() return mb.purge() } @@ -196,12 +205,10 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro // Loop over mailboxes for _, inf3 := range infos3 { if inf3.IsDir() { - mbdir := inf3.Name() - mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) - idx := filepath.Join(mbpath, indexFileName) - mb := &mbox{store: fs, dirName: mbdir, path: mbpath, - indexPath: idx} + mb := fs.mboxFromHash(inf3.Name()) + mb.RLock() msgs, err := mb.getMessages() + mb.RUnlock() if err != nil { return err } @@ -217,265 +224,40 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro return nil } -// LockFor returns the RWMutex for this mailbox, or an error. -func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { - name, err := policy.ParseMailboxName(emailAddress) - if err != nil { - return nil, err - } - hash := stringutil.HashMailboxName(name) - return fs.hashLock.Get(hash), nil -} - -// NewMessage is temproary until #69 MessageData refactor -func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { - mb, err := fs.mbox(mailbox) - if err != nil { - return nil, err - } - return mb.newMessage() -} - // mbox returns the named mailbox. func (fs *Store) mbox(mailbox string) (*mbox, error) { name, err := policy.ParseMailboxName(mailbox) if err != nil { return nil, err } - dir := stringutil.HashMailboxName(name) - s1 := dir[0:3] - s2 := dir[0:6] - path := filepath.Join(fs.mailPath, s1, s2, dir) + hash := stringutil.HashMailboxName(name) + s1 := hash[0:3] + s2 := hash[0:6] + path := filepath.Join(fs.mailPath, s1, s2, hash) indexPath := filepath.Join(path, indexFileName) - - return &mbox{store: fs, name: name, dirName: dir, path: path, - indexPath: indexPath}, nil + return &mbox{ + RWMutex: fs.hashLock.Get(hash), + store: fs, + name: name, + dirName: hash, + path: path, + indexPath: indexPath, + }, nil } -// mbox manages the mail for a specific user and correlates to a particular directory on disk. -type mbox struct { - store *Store - name string - dirName string - path string - indexLoaded bool - indexPath string - messages []*Message -} - -// getMessages scans the mailbox directory for .gob files and decodes them into -// a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return nil, err - } +// mboxFromPath constructs a mailbox based on name hash. +func (fs *Store) mboxFromHash(hash string) *mbox { + s1 := hash[0:3] + s2 := hash[0:6] + path := filepath.Join(fs.mailPath, s1, s2, hash) + indexPath := filepath.Join(path, indexFileName) + return &mbox{ + RWMutex: fs.hashLock.Get(hash), + store: fs, + dirName: hash, + path: path, + indexPath: indexPath, } - messages := make([]storage.StoreMessage, len(mb.messages)) - for i, m := range mb.messages { - messages[i] = m - } - return messages, nil -} - -// getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return nil, err - } - } - if id == "latest" && len(mb.messages) != 0 { - return mb.messages[len(mb.messages)-1], nil - } - for _, m := range mb.messages { - if m.Fid == id { - return m, nil - } - } - return nil, storage.ErrNotExist -} - -// removeMessage deletes the message off disk and removes it from the index. -func (mb *mbox) removeMessage(id string) error { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return err - } - } - var msg *Message - for i, m := range mb.messages { - if id == m.ID() { - msg = m - // Slice around message we are deleting - mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) - break - } - } - if msg == nil { - return storage.ErrNotExist - } - if err := mb.writeIndex(); err != nil { - return err - } - if len(mb.messages) == 0 { - // This was the last message, thus writeIndex() has removed the entire - // directory; we don't need to delete the raw file. - return nil - } - // There are still messages in the index - log.Tracef("Deleting %v", msg.rawPath()) - return os.Remove(msg.rawPath()) -} - -// purge deletes all messages in this mailbox. -func (mb *mbox) purge() error { - mb.messages = mb.messages[:0] - return mb.writeIndex() -} - -// readIndex loads the mailbox index data from disk -func (mb *mbox) readIndex() error { - // Clear message slice, open index - mb.messages = mb.messages[:0] - // Lock for reading - indexMx.RLock() - defer indexMx.RUnlock() - // Check if index exists - if _, err := os.Stat(mb.indexPath); err != nil { - // Does not exist, but that's not an error in our world - log.Tracef("Index %v does not exist (yet)", mb.indexPath) - mb.indexLoaded = true - return nil - } - file, err := os.Open(mb.indexPath) - if err != nil { - return err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) - } - }() - // Decode gob data - dec := gob.NewDecoder(bufio.NewReader(file)) - name := "" - if err = dec.Decode(&name); err != nil { - return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) - } - mb.name = name - for { - // Load messages until EOF - msg := &Message{} - if err = dec.Decode(msg); err != nil { - if err == io.EOF { - break - } - return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) - } - msg.mailbox = mb - mb.messages = append(mb.messages, msg) - } - mb.indexLoaded = true - return nil -} - -// writeIndex overwrites the index on disk with the current mailbox data -func (mb *mbox) writeIndex() error { - // Lock for writing - indexMx.Lock() - defer indexMx.Unlock() - if len(mb.messages) > 0 { - // Ensure mailbox directory exists - if err := mb.createDir(); err != nil { - return err - } - // Open index for writing - file, err := os.Create(mb.indexPath) - if err != nil { - return err - } - writer := bufio.NewWriter(file) - // Write each message and then flush - enc := gob.NewEncoder(writer) - if err = enc.Encode(mb.name); err != nil { - _ = file.Close() - return err - } - for _, m := range mb.messages { - if err = enc.Encode(m); err != nil { - _ = file.Close() - return err - } - } - if err := writer.Flush(); err != nil { - _ = file.Close() - return err - } - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) - return err - } - } else { - // No messages, delete index+maildir - log.Tracef("Removing mailbox %v", mb.path) - return mb.removeDir() - } - return nil -} - -// createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *mbox) createDir() error { - dirMx.Lock() - defer dirMx.Unlock() - if _, err := os.Stat(mb.path); err != nil { - if err := os.MkdirAll(mb.path, 0770); err != nil { - log.Errorf("Failed to create directory %v, %v", mb.path, err) - return err - } - } - return nil -} - -// removeDir removes the mailbox, plus empty higher level directories -func (mb *mbox) removeDir() error { - dirMx.Lock() - defer dirMx.Unlock() - // remove mailbox dir, including index file - if err := os.RemoveAll(mb.path); err != nil { - return err - } - // remove parents if empty - dir := filepath.Dir(mb.path) - if removeDirIfEmpty(dir) { - removeDirIfEmpty(filepath.Dir(dir)) - } - return nil -} - -// removeDirIfEmpty will remove the specified directory if it contains no files or directories. -// Caller should hold dirMx. Returns true if dir was removed. -func removeDirIfEmpty(path string) (removed bool) { - f, err := os.Open(path) - if err != nil { - return false - } - files, err := f.Readdirnames(0) - _ = f.Close() - if err != nil { - return false - } - if len(files) > 0 { - // Dir not empty - return false - } - log.Tracef("Removing dir %v", path) - err = os.Remove(path) - if err != nil { - log.Errorf("Failed to remove %q: %v", path, err) - return false - } - return true } // generatePrefix converts a Time object into the ISO style format we use diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go new file mode 100644 index 0000000..ea6d7f4 --- /dev/null +++ b/pkg/storage/file/mbox.go @@ -0,0 +1,242 @@ +package file + +import ( + "bufio" + "encoding/gob" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// mbox manages the mail for a specific user and correlates to a particular directory on disk. +// mbox methods are not thread safe, mbox.RWMutex must be held prior to calling. +type mbox struct { + *sync.RWMutex + store *Store + name string + dirName string + path string + indexLoaded bool + indexPath string + messages []*Message +} + +// getMessages scans the mailbox directory for .gob files and decodes them into +// a slice of Message objects. +func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return nil, err + } + } + messages := make([]storage.StoreMessage, len(mb.messages)) + for i, m := range mb.messages { + messages[i] = m + } + return messages, nil +} + +// getMessage decodes a single message by ID and returns a Message object. +func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return nil, err + } + } + if id == "latest" && len(mb.messages) != 0 { + return mb.messages[len(mb.messages)-1], nil + } + for _, m := range mb.messages { + if m.Fid == id { + return m, nil + } + } + return nil, storage.ErrNotExist +} + +// removeMessage deletes the message off disk and removes it from the index. +func (mb *mbox) removeMessage(id string) error { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return err + } + } + var msg *Message + for i, m := range mb.messages { + if id == m.ID() { + msg = m + // Slice around message we are deleting + mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) + break + } + } + if msg == nil { + return storage.ErrNotExist + } + if err := mb.writeIndex(); err != nil { + return err + } + if len(mb.messages) == 0 { + // This was the last message, thus writeIndex() has removed the entire + // directory; we don't need to delete the raw file. + return nil + } + // There are still messages in the index + log.Tracef("Deleting %v", msg.rawPath()) + return os.Remove(msg.rawPath()) +} + +// purge deletes all messages in this mailbox. +func (mb *mbox) purge() error { + mb.messages = mb.messages[:0] + return mb.writeIndex() +} + +// readIndex loads the mailbox index data from disk +func (mb *mbox) readIndex() error { + // Clear message slice, open index + mb.messages = mb.messages[:0] + // Lock for reading + indexMx.RLock() + defer indexMx.RUnlock() + // Check if index exists + if _, err := os.Stat(mb.indexPath); err != nil { + // Does not exist, but that's not an error in our world + log.Tracef("Index %v does not exist (yet)", mb.indexPath) + mb.indexLoaded = true + return nil + } + file, err := os.Open(mb.indexPath) + if err != nil { + return err + } + defer func() { + if err := file.Close(); err != nil { + log.Errorf("Failed to close %q: %v", mb.indexPath, err) + } + }() + // Decode gob data + dec := gob.NewDecoder(bufio.NewReader(file)) + name := "" + if err = dec.Decode(&name); err != nil { + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + mb.name = name + for { + // Load messages until EOF + msg := &Message{} + if err = dec.Decode(msg); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + msg.mailbox = mb + mb.messages = append(mb.messages, msg) + } + mb.indexLoaded = true + return nil +} + +// writeIndex overwrites the index on disk with the current mailbox data +func (mb *mbox) writeIndex() error { + // Lock for writing + indexMx.Lock() + defer indexMx.Unlock() + if len(mb.messages) > 0 { + // Ensure mailbox directory exists + if err := mb.createDir(); err != nil { + return err + } + // Open index for writing + file, err := os.Create(mb.indexPath) + if err != nil { + return err + } + writer := bufio.NewWriter(file) + // Write each message and then flush + enc := gob.NewEncoder(writer) + if err = enc.Encode(mb.name); err != nil { + _ = file.Close() + return err + } + for _, m := range mb.messages { + if err = enc.Encode(m); err != nil { + _ = file.Close() + return err + } + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + log.Errorf("Failed to close %q: %v", mb.indexPath, err) + return err + } + } else { + // No messages, delete index+maildir + log.Tracef("Removing mailbox %v", mb.path) + return mb.removeDir() + } + return nil +} + +// createDir checks for the presence of the path for this mailbox, creates it if needed +func (mb *mbox) createDir() error { + dirMx.Lock() + defer dirMx.Unlock() + if _, err := os.Stat(mb.path); err != nil { + if err := os.MkdirAll(mb.path, 0770); err != nil { + log.Errorf("Failed to create directory %v, %v", mb.path, err) + return err + } + } + return nil +} + +// removeDir removes the mailbox, plus empty higher level directories +func (mb *mbox) removeDir() error { + dirMx.Lock() + defer dirMx.Unlock() + // remove mailbox dir, including index file + if err := os.RemoveAll(mb.path); err != nil { + return err + } + // remove parents if empty + dir := filepath.Dir(mb.path) + if removeDirIfEmpty(dir) { + removeDirIfEmpty(filepath.Dir(dir)) + } + return nil +} + +// removeDirIfEmpty will remove the specified directory if it contains no files or directories. +// Caller should hold dirMx. Returns true if dir was removed. +func removeDirIfEmpty(path string) (removed bool) { + f, err := os.Open(path) + if err != nil { + return false + } + files, err := f.Readdirnames(0) + _ = f.Close() + if err != nil { + return false + } + if len(files) > 0 { + // Dir not empty + return false + } + log.Tracef("Removing dir %v", path) + err = os.Remove(path) + if err != nil { + log.Errorf("Failed to remove %q: %v", path, err) + return false + } + return true +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 62d4972..f8bcaef 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -5,7 +5,6 @@ import ( "errors" "io" "net/mail" - "sync" "time" ) @@ -26,8 +25,6 @@ type Store interface { PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error VisitMailboxes(f func([]StoreMessage) (cont bool)) error - // LockFor is a temporary hack to fix #77 until Datastore revamp - LockFor(emailAddress string) (*sync.RWMutex, error) } // StoreMessage represents a message to be stored, or returned from a storage implementation. diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 52c9b00..2c45b10 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -2,7 +2,6 @@ package test import ( "errors" - "sync" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -82,12 +81,6 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e return nil } -// LockFor mock function returns a new RWMutex, never errors. -// TODO(#69) remove -func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { - return &sync.RWMutex{}, nil -} - // MessageDeleted returns true if the specified message was deleted func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { _, ok := s.deleted[m] From dc4db59211e142b3195914cdcb8f7e8d2a0a6eee Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 14:41:03 -0700 Subject: [PATCH 23/82] smtp: Don't require MIME headers for metadata This was a regression, will again fall back to MAIL FROM/RCPT TO data. --- pkg/server/smtp/handler.go | 10 ++++++---- pkg/server/smtp/handler_test.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 1848fe2..686f94a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "net/mail" "regexp" "strconv" "strings" @@ -408,13 +409,14 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b } from, err := env.AddressList("From") if err != nil { - ss.logError("Failed to get From address: %v", err) - return false + from = []*mail.Address{{Address: ss.from}} } to, err := env.AddressList("To") if err != nil { - ss.logError("Failed to get To addresses: %v", err) - return false + to = make([]*mail.Address, len(ss.recipients)) + for i, torecip := range ss.recipients { + to[i] = &torecip.Address + } } // Generate Received header. stamp := time.Now().Format(timeStampFormat) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 6242b20..49100d2 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -200,7 +200,7 @@ func TestMailState(t *testing.T) { {"MAIL FROM:", 250}, {"RCPT TO:", 250}, {"DATA", 354}, - {".", 451}, + {".", 250}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -247,7 +247,6 @@ func TestDataState(t *testing.T) { pipe := setupSMTPSession(server) c := textproto.NewConn(pipe) - // Get us into DATA state if code, _, err := c.ReadCodeLine(220); err != nil { t.Errorf("Expected a 220 greeting, got %v", code) } @@ -274,6 +273,33 @@ Hi! t.Errorf("Expected a 250 greeting, got %v", code) } + // Test with no useful headers. + pipe = setupSMTPSession(server) + c = textproto.NewConn(pipe) + if code, _, err := c.ReadCodeLine(220); err != nil { + t.Errorf("Expected a 220 greeting, got %v", code) + } + script = []scriptStep{ + {"HELO localhost", 250}, + {"MAIL FROM:", 250}, + {"RCPT TO:", 250}, + {"DATA", 354}, + } + if err := playScriptAgainst(t, c, script); err != nil { + t.Error(err) + } + // Send a message + body = `X-Useless-Header: true + +Hi! Can you still deliver this? +` + dw = c.DotWriter() + _, _ = io.WriteString(dw, body) + _ = dw.Close() + if code, _, err := c.ReadCodeLine(250); err != nil { + t.Errorf("Expected a 250 greeting, got %v", code) + } + if t.Failed() { // Wait for handler to finish logging time.Sleep(2 * time.Second) From a22412f65e737cb0dc182098990a07a1fd9b8503 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 15:17:44 -0700 Subject: [PATCH 24/82] manager: Add MailboxForAddress(), calls policy pkg #84 --- pkg/message/manager.go | 7 +++++++ pkg/rest/apiv1_controller.go | 11 +++++------ pkg/rest/socketv1_controller.go | 3 +-- pkg/test/manager.go | 6 ++++++ pkg/webui/mailbox_controller.go | 17 ++++++++--------- pkg/webui/root_controller.go | 3 +-- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 5782273..1c4e04e 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -4,6 +4,7 @@ import ( "io" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -14,6 +15,7 @@ type Manager interface { PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error SourceReader(mailbox, id string) (io.ReadCloser, error) + MailboxForAddress(address string) (string, error) } // StoreManager is a message Manager backed by the storage.Store. @@ -72,6 +74,11 @@ func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { return sm.RawReader() } +// MailboxForAddress parses an email address to return the canonical mailbox name. +func (s *StoreManager) MailboxForAddress(mailbox string) (string, error) { + return policy.ParseMailboxName(mailbox) +} + // makeMetadata populates Metadata from a StoreMessage. func makeMetadata(m storage.StoreMessage) *Metadata { return &Metadata{ diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 2b10eb0..c463ecf 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -11,7 +11,6 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -21,7 +20,7 @@ import ( // MailboxListV1 renders a list of messages in a mailbox func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -51,7 +50,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -101,7 +100,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // MailboxPurgeV1 deletes all messages from a mailbox func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -119,7 +118,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -143,7 +142,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index 7614ac1..d0ceddd 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -7,7 +7,6 @@ import ( "github.com/gorilla/websocket" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" ) @@ -173,7 +172,7 @@ func MonitorAllMessagesV1( // notifies the client of messages received by a particular mailbox. func MonitorMailboxMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/test/manager.go b/pkg/test/manager.go index 47c87a8..bf8d9ad 100644 --- a/pkg/test/manager.go +++ b/pkg/test/manager.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -51,3 +52,8 @@ func (m *ManagerStub) GetMetadata(mailbox string) ([]*message.Metadata, error) { } return metas, nil } + +// MailboxForAddress invokes policy.ParseMailboxName. +func (m *ManagerStub) MailboxForAddress(address string) (string, error) { + return policy.ParseMailboxName(address) +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 2af9ef1..d876b3a 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -8,7 +8,6 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/webui/sanitize" @@ -25,7 +24,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err = policy.ParseMailboxName(name) + name, err = ctx.Manager.MailboxForAddress(name) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -52,7 +51,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -68,7 +67,7 @@ func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // MailboxList renders a list of messages in a mailbox. Renders a partial func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -90,7 +89,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -131,7 +130,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -159,7 +158,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -183,7 +182,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -225,7 +224,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // MailboxViewAttach sends the attachment to the client for online viewing func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) diff --git a/pkg/webui/root_controller.go b/pkg/webui/root_controller.go index 21d4dd3..0ea5780 100644 --- a/pkg/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" ) @@ -58,7 +57,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Conte http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) From f953bcf4bbb7c9281f3b93f7323e74be8da4f500 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 16:54:29 -0700 Subject: [PATCH 25/82] smtp: Move delivery into message.Manager for #69 --- cmd/inbucket/main.go | 31 ++++++-------- pkg/message/manager.go | 68 +++++++++++++++++++++++++++++++ pkg/server/smtp/handler.go | 71 +++++---------------------------- pkg/server/smtp/handler_test.go | 8 ++-- pkg/server/smtp/listener.go | 17 ++++---- 5 files changed, 102 insertions(+), 93 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index ff5a313..0de0b07 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -115,32 +115,27 @@ func main() { } } - // Create message hub + // Configure internal services. msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) - - // Setup our datastore dscfg := config.GetDataStoreConfig() - ds := file.New(dscfg) - retentionScanner := storage.NewRetentionScanner(dscfg, ds, shutdownChan) + store := file.New(dscfg) + apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} + mmanager := &message.StoreManager{Store: store, Hub: msgHub} + // Start Retention scanner. + retentionScanner := storage.NewRetentionScanner(dscfg, store, shutdownChan) retentionScanner.Start() - - // Start HTTP server - mm := &message.StoreManager{Store: ds} - web.Initialize(config.GetWebConfig(), shutdownChan, mm, msgHub) + // Start HTTP server. + web.Initialize(config.GetWebConfig(), shutdownChan, mmanager, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) - - // Start POP3 server - pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, ds) + // Start POP3 server. + pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, store) go pop3Server.Start(rootCtx) - - // Startup SMTP server - apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} - smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, apolicy, msgHub) + // Start SMTP server. + smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, mmanager, apolicy) go smtpServer.Start(rootCtx) - - // Loop forever waiting for signals or shutdown channel + // Loop forever waiting for signals or shutdown channel. signalLoop: for { select { diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 1c4e04e..c9b74f1 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -1,15 +1,28 @@ package message import ( + "bytes" "io" + "net/mail" + "strings" + "time" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // Manager is the interface controllers use to interact with messages. type Manager interface { + Deliver( + to *policy.Recipient, + from string, + recipients []*policy.Recipient, + prefix string, + content []byte, + ) (id string, err error) GetMetadata(mailbox string) ([]*Metadata, error) GetMessage(mailbox, id string) (*Message, error) PurgeMessages(mailbox string) error @@ -21,6 +34,61 @@ type Manager interface { // StoreManager is a message Manager backed by the storage.Store. type StoreManager struct { Store storage.Store + Hub *msghub.Hub +} + +// Deliver submits a new message to the store. +func (s *StoreManager) Deliver( + to *policy.Recipient, + from string, + recipients []*policy.Recipient, + prefix string, + content []byte, +) (string, error) { + // TODO enmime is too heavy for this step, only need header + env, err := enmime.ReadEnvelope(bytes.NewReader(content)) + if err != nil { + return "", err + } + fromaddr, err := env.AddressList("From") + if err != nil || len(fromaddr) == 0 { + fromaddr = []*mail.Address{{Address: from}} + } + toaddr, err := env.AddressList("To") + if err != nil { + toaddr = make([]*mail.Address, len(recipients)) + for i, torecip := range recipients { + toaddr[i] = &torecip.Address + } + } + delivery := &Delivery{ + Meta: Metadata{ + Mailbox: to.Mailbox, + From: fromaddr[0], + To: toaddr, + Date: time.Now(), + Subject: env.GetHeader("Subject"), + }, + Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(content)), + } + id, err := s.Store.AddMessage(delivery) + if err != nil { + return "", err + } + if s.Hub != nil { + // Broadcast message information. + broadcast := msghub.Message{ + Mailbox: to.Mailbox, + ID: id, + From: delivery.From().String(), + To: stringutil.StringAddressList(delivery.To()), + Subject: delivery.Subject(), + Date: delivery.Date(), + Size: delivery.Size(), + } + s.Hub.Dispatch(broadcast) + } + return id, nil } // GetMetadata returns a slice of metadata for the specified mailbox. diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 686f94a..4d6ee33 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -6,18 +6,13 @@ import ( "fmt" "io" "net" - "net/mail" "regexp" "strconv" "strings" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/message" - "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/policy" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) // State tracks the current mode of our SMTP state machine @@ -370,9 +365,18 @@ func (ss *Session) dataHandler() { } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete. + tstamp := time.Now().Format(timeStampFormat) for _, recip := range ss.recipients { if recip.ShouldStore() { - if ok := ss.deliverMessage(recip, msgBuf.Bytes()); !ok { + // Generate Received header. + prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address.Address, + tstamp) + // Deliver message. + _, err := ss.server.manager.Deliver( + recip, ss.from, ss.recipients, prefix, msgBuf.Bytes()) + if err != nil { + ss.logError("delivery for %v: %v", recip.LocalPart, err) ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return @@ -385,7 +389,7 @@ func (ss *Session) dataHandler() { ss.reset() return } - // RFC says remove leading periods from input. + // RFC: remove leading periods from DATA. if len(lineBuf) > 0 && lineBuf[0] == '.' { lineBuf = lineBuf[1:] } @@ -399,59 +403,6 @@ func (ss *Session) dataHandler() { } } -// deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { - // TODO replace with something that only reads header? - env, err := enmime.ReadEnvelope(bytes.NewReader(content)) - if err != nil { - ss.logError("Failed to parse message for %q: %v", recip.LocalPart, err) - return false - } - from, err := env.AddressList("From") - if err != nil { - from = []*mail.Address{{Address: ss.from}} - } - to, err := env.AddressList("To") - if err != nil { - to = make([]*mail.Address, len(ss.recipients)) - for i, torecip := range ss.recipients { - to[i] = &torecip.Address - } - } - // Generate Received header. - stamp := time.Now().Format(timeStampFormat) - recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) - delivery := &message.Delivery{ - Meta: message.Metadata{ - Mailbox: recip.Mailbox, - From: from[0], - To: to, - Date: time.Now(), - Subject: env.GetHeader("Subject"), - }, - Reader: io.MultiReader(recd, bytes.NewReader(content)), - } - id, err := ss.server.dataStore.AddMessage(delivery) - if err != nil { - ss.logError("Failed to store message for %q: %s", recip.LocalPart, err) - return false - } - // Broadcast message information. - // TODO this belongs in message pkg. - broadcast := msghub.Message{ - Mailbox: recip.Mailbox, - ID: id, - From: delivery.From().String(), - To: stringutil.StringAddressList(delivery.To()), - Subject: delivery.Subject(), - Date: delivery.Date(), - Size: delivery.Size(), - } - ss.server.msgHub.Dispatch(broadcast) - return true -} - func (ss *Session) enterState(state State) { ss.state = state ss.logTrace("Entering state %v", state) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 49100d2..283c6d1 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -2,7 +2,6 @@ package smtp import ( "bytes" - "context" "fmt" "io" @@ -14,7 +13,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" @@ -379,13 +378,12 @@ func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown f // Create a server, don't start it shutdownChan := make(chan bool) - ctx, cancel := context.WithCancel(context.Background()) teardown = func() { close(shutdownChan) - cancel() } apolicy := &policy.Addressing{Config: cfg} - s = NewServer(cfg, shutdownChan, ds, apolicy, msghub.New(ctx, 100)) + manager := &message.StoreManager{Store: ds} + s = NewServer(cfg, shutdownChan, manager, apolicy) return s, buf, teardown } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 0e795b7..959b6f5 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -12,9 +12,8 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/policy" - "github.com/jhillyerd/inbucket/pkg/storage" ) func init() { @@ -49,10 +48,9 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - apolicy *policy.Addressing // Address policy - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info + apolicy *policy.Addressing // Address policy. + globalShutdown chan bool // Shuts down Inbucket. + manager message.Manager // Used to deliver messages. // State listener net.Listener // Incoming network connections @@ -84,9 +82,9 @@ var ( func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, - ds storage.Store, + manager message.Manager, apolicy *policy.Addressing, - msgHub *msghub.Hub) *Server { +) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), domain: cfg.Domain, @@ -96,9 +94,8 @@ func NewServer( maxMessageBytes: cfg.MaxMessageBytes, storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, - dataStore: ds, + manager: manager, apolicy: apolicy, - msgHub: msgHub, waitgroup: new(sync.WaitGroup), } } From 30a329c0d31a9be255f7083eeeca994708604d05 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 17:56:06 -0700 Subject: [PATCH 26/82] Renames, closes #69 - storage: rename StoreMessage to Message - storage: rename Message.RawReader() to Source() --- pkg/message/manager.go | 14 +++++++------- pkg/message/message.go | 6 +++--- pkg/server/pop3/handler.go | 30 +++++++++++++++--------------- pkg/storage/file/fmessage.go | 4 ++-- pkg/storage/file/fstore.go | 10 +++++----- pkg/storage/file/fstore_test.go | 2 +- pkg/storage/file/mbox.go | 6 +++--- pkg/storage/retention.go | 2 +- pkg/storage/retention_test.go | 6 +++--- pkg/storage/storage.go | 14 +++++++------- pkg/test/storage.go | 20 ++++++++++---------- pkg/test/storage_suite.go | 6 +++--- 12 files changed, 60 insertions(+), 60 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index c9b74f1..2a8bb47 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -43,10 +43,10 @@ func (s *StoreManager) Deliver( from string, recipients []*policy.Recipient, prefix string, - content []byte, + source []byte, ) (string, error) { // TODO enmime is too heavy for this step, only need header - env, err := enmime.ReadEnvelope(bytes.NewReader(content)) + env, err := enmime.ReadEnvelope(bytes.NewReader(source)) if err != nil { return "", err } @@ -69,7 +69,7 @@ func (s *StoreManager) Deliver( Date: time.Now(), Subject: env.GetHeader("Subject"), }, - Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(content)), + Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(source)), } id, err := s.Store.AddMessage(delivery) if err != nil { @@ -110,7 +110,7 @@ func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { if err != nil { return nil, err } - r, err := sm.RawReader() + r, err := sm.Source() if err != nil { return nil, err } @@ -139,7 +139,7 @@ func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { if err != nil { return nil, err } - return sm.RawReader() + return sm.Source() } // MailboxForAddress parses an email address to return the canonical mailbox name. @@ -147,8 +147,8 @@ func (s *StoreManager) MailboxForAddress(mailbox string) (string, error) { return policy.ParseMailboxName(mailbox) } -// makeMetadata populates Metadata from a StoreMessage. -func makeMetadata(m storage.StoreMessage) *Metadata { +// makeMetadata populates Metadata from a storage.Message. +func makeMetadata(m storage.Message) *Metadata { return &Metadata{ Mailbox: m.Mailbox(), ID: m.ID(), diff --git a/pkg/message/message.go b/pkg/message/message.go index 3994ca3..7f8bec0 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -34,7 +34,7 @@ type Delivery struct { Reader io.Reader } -var _ storage.StoreMessage = &Delivery{} +var _ storage.Message = &Delivery{} // Mailbox getter. func (d *Delivery) Mailbox() string { @@ -71,7 +71,7 @@ func (d *Delivery) Size() int64 { return d.Meta.Size } -// RawReader contains the raw content of the message. -func (d *Delivery) RawReader() (io.ReadCloser, error) { +// Source contains the raw content of the message. +func (d *Delivery) Source() (io.ReadCloser, error) { return ioutil.NopCloser(d.Reader), nil } diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index c022619..f8229ca 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,17 +57,17 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - messages []storage.StoreMessage // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + messages []storage.Message // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -415,8 +415,8 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg storage.StoreMessage) { - reader, err := msg.RawReader() +func (ses *Session) sendMessage(msg storage.Message) { + reader, err := msg.Source() if err != nil { ses.logError("Failed to read message for RETR command") ses.send("-ERR Failed to RETR that message, internal error") @@ -448,8 +448,8 @@ func (ses *Session) sendMessage(msg storage.StoreMessage) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg storage.StoreMessage, lineCount int) { - reader, err := msg.RawReader() +func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { + reader, err := msg.Source() if err != nil { ses.logError("Failed to read message for RETR command") ses.send("-ERR Failed to RETR that message, internal error") diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index cf06d5c..cf375b4 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -90,8 +90,8 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// RawReader opens the .raw portion of a Message as an io.ReadCloser -func (m *Message) RawReader() (reader io.ReadCloser, err error) { +// Source opens the .raw portion of a Message as an io.ReadCloser +func (m *Message) Source() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index f6a60ec..ebe5c7e 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -75,14 +75,14 @@ func New(cfg config.DataStoreConfig) storage.Store { } // AddMessage adds a message to the specified mailbox. -func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { +func (fs *Store) AddMessage(m storage.Message) (id string, err error) { mb, err := fs.mbox(m.Mailbox()) if err != nil { return "", err } mb.Lock() defer mb.Unlock() - r, err := m.RawReader() + r, err := m.Source() if err != nil { return "", err } @@ -136,7 +136,7 @@ func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { } // GetMessage returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { +func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -147,7 +147,7 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { } // GetMessages returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { +func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -181,7 +181,7 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { +func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return err diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 18b1fe7..589bf18 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -133,7 +133,7 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.RawReader() + _, err = msg.Source() assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index ea6d7f4..cb5c1b9 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -28,13 +28,13 @@ type mbox struct { // getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { +func (mb *mbox) getMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.StoreMessage, len(mb.messages)) + messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -42,7 +42,7 @@ func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { } // getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { +func (mb *mbox) getMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 2da706e..6c7adb0 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -119,7 +119,7 @@ func (rs *RetentionScanner) DoScan() error { cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 // Loop over all mailboxes. - err := rs.ds.VisitMailboxes(func(messages []StoreMessage) bool { + err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { log.Tracef("Purging expired message %v/%v", msg.Mailbox(), msg.ID()) diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index a49cc59..234a377 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -37,13 +37,13 @@ func TestDoRetentionScan(t *testing.T) { t.Error(err) } // Delete should not have been called on new messages - for _, m := range []storage.StoreMessage{new1, new2, new3} { + for _, m := range []storage.Message{new1, new2, new3} { if ds.MessageDeleted(m) { t.Errorf("Expected %v to be present, was deleted", m.ID()) } } // Delete should have been called once on old messages - for _, m := range []storage.StoreMessage{old1, old2, old3} { + for _, m := range []storage.Message{old1, old2, old3} { if !ds.MessageDeleted(m) { t.Errorf("Expected %v to be deleted, was present", m.ID()) } @@ -51,7 +51,7 @@ func TestDoRetentionScan(t *testing.T) { } // stubMessage creates a message stub of a specific age -func stubMessage(mailbox string, ageHours int) storage.StoreMessage { +func stubMessage(mailbox string, ageHours int) storage.Message { return &message.Delivery{ Meta: message.Metadata{ Mailbox: mailbox, diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f8bcaef..edc4afd 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -19,22 +19,22 @@ var ( // Store is the interface Inbucket uses to interact with storage implementations. type Store interface { // AddMessage stores the message, message ID and Size will be ignored. - AddMessage(message StoreMessage) (id string, err error) - GetMessage(mailbox, id string) (StoreMessage, error) - GetMessages(mailbox string) ([]StoreMessage, error) + AddMessage(message Message) (id string, err error) + GetMessage(mailbox, id string) (Message, error) + GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error - VisitMailboxes(f func([]StoreMessage) (cont bool)) error + VisitMailboxes(f func([]Message) (cont bool)) error } -// StoreMessage represents a message to be stored, or returned from a storage implementation. -type StoreMessage interface { +// Message represents a message to be stored, or returned from a storage implementation. +type Message interface { Mailbox() string ID() string From() *mail.Address To() []*mail.Address Date() time.Time Subject() string - RawReader() (reader io.ReadCloser, err error) + Source() (io.ReadCloser, error) Size() int64 } diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 2c45b10..b52445d 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -9,20 +9,20 @@ import ( // StoreStub stubs storage.Store for testing. type StoreStub struct { storage.Store - mailboxes map[string][]storage.StoreMessage - deleted map[storage.StoreMessage]struct{} + mailboxes map[string][]storage.Message + deleted map[storage.Message]struct{} } // NewStore creates a new StoreStub. func NewStore() *StoreStub { return &StoreStub{ - mailboxes: make(map[string][]storage.StoreMessage), - deleted: make(map[storage.StoreMessage]struct{}), + mailboxes: make(map[string][]storage.Message), + deleted: make(map[storage.Message]struct{}), } } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { +func (s *StoreStub) AddMessage(m storage.Message) (id string, err error) { mb := m.Mailbox() msgs := s.mailboxes[mb] s.mailboxes[mb] = append(msgs, m) @@ -30,7 +30,7 @@ func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { } // GetMessage gets a message by ID from the specified mailbox. -func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) { +func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { if mailbox == "messageerr" { return nil, errors.New("internal error") } @@ -43,7 +43,7 @@ func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) } // GetMessages gets all the messages for the specified mailbox. -func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) { +func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { if mailbox == "messageserr" { return nil, errors.New("internal error") } @@ -54,7 +54,7 @@ func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) func (s *StoreStub) RemoveMessage(mailbox, id string) error { mb, ok := s.mailboxes[mailbox] if ok { - var msg storage.StoreMessage + var msg storage.Message for i, m := range mb { if m.ID() == id { msg = m @@ -72,7 +72,7 @@ func (s *StoreStub) RemoveMessage(mailbox, id string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { +func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { for _, v := range s.mailboxes { if !f(v) { return nil @@ -82,7 +82,7 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e } // MessageDeleted returns true if the specified message was deleted -func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { +func (s *StoreStub) MessageDeleted(m storage.Message) bool { _, ok := s.deleted[m] return ok } diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 80e8626..3d167b5 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -138,7 +138,7 @@ func testContent(t *testing.T, store storage.Store) { if err != nil { t.Fatal(err) } - r, err := m.RawReader() + r, err := m.Source() if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func testVisitMailboxes(t *testing.T, ds storage.Store) { deliverMessage(t, ds, name, "New Message", time.Now()) } seen := 0 - err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { + err := ds.VisitMailboxes(func(messages []storage.Message) bool { seen++ count := len(messages) if count != 2 { @@ -317,7 +317,7 @@ func deliverMessage( // getAndCountMessages is a test helper that expects to receive count messages or fails the test, it // also checks return error. -func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.StoreMessage { +func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.Message { t.Helper() msgs, err := s.GetMessages(mailbox) if err != nil { From 5cb07d5780f7e6ff94036a9e8b9788a6b000f309 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 12:08:40 -0700 Subject: [PATCH 27/82] rest: Refactor JSON result value testing --- pkg/rest/apiv1_controller_test.go | 95 ++++-------- pkg/rest/testutils_test.go | 232 ++++++++++++------------------ 2 files changed, 116 insertions(+), 211 deletions(-) diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index b31cfcb..4906d21 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -6,7 +6,6 @@ import ( "net/mail" "net/textproto" "os" - "strings" "testing" "time" @@ -68,22 +67,6 @@ func TestRestMailboxList(t *testing.T) { } // Test JSON message headers - data1 := &InputMessageData{ - Mailbox: "good", - ID: "0001", - From: "", - To: []string{""}, - Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), - } - data2 := &InputMessageData{ - Mailbox: "good", - ID: "0002", - From: "", - To: []string{""}, - Subject: "subject 2", - Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), - } meta1 := message.Metadata{ Mailbox: "good", ID: "0001", @@ -114,25 +97,6 @@ func TestRestMailboxList(t *testing.T) { } // Check JSON - got := w.Body.String() - testStrings := []string{ - `{"mailbox":"good","id":"0001","from":"\u003cfrom1@host\u003e",` + - `"to":["\u003cto1@host\u003e"],"subject":"subject 1",` + - `"date":"2012-02-01T10:11:12.000000253-00:13","size":0}`, - `{"mailbox":"good","id":"0002","from":"\u003cfrom2@host\u003e",` + - `"to":["\u003cto1@host\u003e"],"subject":"subject 2",` + - `"date":"2012-07-01T10:11:12.000000253-00:11","size":0}`, - } - for _, ts := range testStrings { - t.Run(ts, func(t *testing.T) { - if !strings.Contains(got, ts) { - t.Errorf("got:\n%s\nwant to contain:\n%s", got, ts) - } - }) - } - - // Check JSON - // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result []interface{} if err := dec.Decode(&result); err != nil { @@ -141,18 +105,21 @@ func TestRestMailboxList(t *testing.T) { if len(result) != 2 { t.Fatalf("Expected 2 results, got %v", len(result)) } - if errors := data1.CompareToJSONHeaderMap(result[0]); len(errors) > 0 { - t.Logf("%v", result[0]) - for _, e := range errors { - t.Error(e) - } - } - if errors := data2.CompareToJSONHeaderMap(result[1]); len(errors) > 0 { - t.Logf("%v", result[1]) - for _, e := range errors { - t.Error(e) - } - } + + decodedStringEquals(t, result, "[0]/mailbox", "good") + decodedStringEquals(t, result, "[0]/id", "0001") + decodedStringEquals(t, result, "[0]/from", "") + decodedStringEquals(t, result, "[0]/to/[0]", "") + decodedStringEquals(t, result, "[0]/subject", "subject 1") + decodedStringEquals(t, result, "[0]/date", "2012-02-01T10:11:12.000000253-00:13") + decodedNumberEquals(t, result, "[0]/size", 0) + decodedStringEquals(t, result, "[1]/mailbox", "good") + decodedStringEquals(t, result, "[1]/id", "0002") + decodedStringEquals(t, result, "[1]/from", "") + decodedStringEquals(t, result, "[1]/to/[0]", "") + decodedStringEquals(t, result, "[1]/subject", "subject 2") + decodedStringEquals(t, result, "[1]/date", "2012-07-01T10:11:12.000000253-00:11") + decodedNumberEquals(t, result, "[1]/size", 0) if t.Failed() { // Wait for handler to finish logging @@ -225,19 +192,6 @@ func TestRestMessage(t *testing.T) { }, }, } - data1 := &InputMessageData{ - Mailbox: "good", - ID: "0001", - From: "", - Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), - Header: mail.Header{ - "To": []string{"fred@fish.com", "keyword@nsa.gov"}, - "From": []string{"noreply@inbucket.org"}, - }, - Text: "This is some text", - HTML: "This is some HTML", - } mm.AddMessage("good", msg1) // Check return code @@ -251,19 +205,24 @@ func TestRestMessage(t *testing.T) { } // Check JSON - // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result map[string]interface{} if err := dec.Decode(&result); err != nil { t.Errorf("Failed to decode JSON: %v", err) } - if errors := data1.CompareToJSONMessageMap(result); len(errors) > 0 { - t.Logf("%v", result) - for _, e := range errors { - t.Error(e) - } - } + decodedStringEquals(t, result, "mailbox", "good") + decodedStringEquals(t, result, "id", "0001") + decodedStringEquals(t, result, "from", "") + decodedStringEquals(t, result, "to/[0]", "") + decodedStringEquals(t, result, "subject", "subject 1") + decodedStringEquals(t, result, "date", "2012-02-01T10:11:12.000000253-00:13") + decodedNumberEquals(t, result, "size", 0) + decodedStringEquals(t, result, "body/text", "This is some text") + decodedStringEquals(t, result, "body/html", "This is some HTML") + decodedStringEquals(t, result, "header/To/[0]", "fred@fish.com") + decodedStringEquals(t, result, "header/To/[1]", "keyword@nsa.gov") + decodedStringEquals(t, result, "header/From/[0]", "noreply@inbucket.org") if t.Failed() { // Wait for handler to finish logging diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 23996a5..587c3b7 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -2,12 +2,12 @@ package rest import ( "bytes" - "fmt" "log" "net/http" "net/http/httptest" - "net/mail" - "time" + "strconv" + "strings" + "testing" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" @@ -15,146 +15,6 @@ import ( "github.com/jhillyerd/inbucket/pkg/server/web" ) -type InputMessageData struct { - Mailbox, ID, From, Subject string - To []string - Date time.Time - Size int - Header mail.Header - HTML, Text string -} - -// isJSONStringEqual is a utility function to return a nicely formatted message when -// comparing a string to a value received from a JSON map. -func isJSONStringEqual(key, expected string, received interface{}) (message string, ok bool) { - if value, ok := received.(string); ok { - if expected == value { - return "", true - } - return fmt.Sprintf("Expected value of key %v to be %q, got %q", key, expected, value), false - } - return fmt.Sprintf("Expected value of key %v to be a string, got %T", key, received), false -} - -// isJSONNumberEqual is a utility function to return a nicely formatted message when -// comparing an float64 to a value received from a JSON map. -func isJSONNumberEqual(key string, expected float64, received interface{}) (message string, ok bool) { - if value, ok := received.(float64); ok { - if expected == value { - return "", true - } - return fmt.Sprintf("Expected %v to be %v, got %v", key, expected, value), false - } - return fmt.Sprintf("Expected %v to be a string, got %T", key, received), false -} - -// CompareToJSONHeaderMap compares InputMessageData to a header map decoded from JSON, -// returning a list of things that did not match. -func (d *InputMessageData) CompareToJSONHeaderMap(json interface{}) (errors []string) { - if m, ok := json.(map[string]interface{}); ok { - if msg, ok := isJSONStringEqual(mailboxKey, d.Mailbox, m[mailboxKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(idKey, d.ID, m[idKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(fromKey, d.From, m[fromKey]); !ok { - errors = append(errors, msg) - } - for i, inputTo := range d.To { - if msg, ok := isJSONStringEqual(toKey, inputTo, m[toKey].([]interface{})[i]); !ok { - errors = append(errors, msg) - } - } - if msg, ok := isJSONStringEqual(subjectKey, d.Subject, m[subjectKey]); !ok { - errors = append(errors, msg) - } - exDate := d.Date.Format("2006-01-02T15:04:05.999999999-07:00") - if msg, ok := isJSONStringEqual(dateKey, exDate, m[dateKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONNumberEqual(sizeKey, float64(d.Size), m[sizeKey]); !ok { - errors = append(errors, msg) - } - return errors - } - panic(fmt.Sprintf("Expected map[string]interface{} in json, got %T", json)) -} - -// CompareToJSONMessageMap compares InputMessageData to a message map decoded from JSON, -// returning a list of things that did not match. -func (d *InputMessageData) CompareToJSONMessageMap(json interface{}) (errors []string) { - // We need to check the same values as header first - errors = d.CompareToJSONHeaderMap(json) - - if m, ok := json.(map[string]interface{}); ok { - // Get nested body map - if m[bodyKey] != nil { - if body, ok := m[bodyKey].(map[string]interface{}); ok { - if msg, ok := isJSONStringEqual(textKey, d.Text, body[textKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(htmlKey, d.HTML, body[htmlKey]); !ok { - errors = append(errors, msg) - } - } else { - panic(fmt.Sprintf("Expected map[string]interface{} in json key %q, got %T", - bodyKey, m[bodyKey])) - } - } else { - errors = append(errors, fmt.Sprintf("Expected body in JSON %q but it was nil", bodyKey)) - } - exDate := d.Date.Format("2006-01-02T15:04:05.999999999-07:00") - if msg, ok := isJSONStringEqual(dateKey, exDate, m[dateKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONNumberEqual(sizeKey, float64(d.Size), m[sizeKey]); !ok { - errors = append(errors, msg) - } - - // Get nested header map - if m[headerKey] != nil { - if header, ok := m[headerKey].(map[string]interface{}); ok { - // Loop over input (expected) header names - for name, keyInputHeaders := range d.Header { - // Make sure expected header name exists in received JSON - if keyOutputVals, ok := header[name]; ok { - if keyOutputHeaders, ok := keyOutputVals.([]interface{}); ok { - // Loop over input (expected) header values - for _, inputHeader := range keyInputHeaders { - hasValue := false - // Look for expected value in received headers - for _, outputHeader := range keyOutputHeaders { - if inputHeader == outputHeader { - hasValue = true - break - } - } - if !hasValue { - errors = append(errors, fmt.Sprintf( - "JSON %v[%q] missing value %q", headerKey, name, inputHeader)) - } - } - } else { - // keyOutputValues was not a slice of interface{} - panic(fmt.Sprintf("Expected []interface{} in %v[%q], got %T", headerKey, - name, keyOutputVals)) - } - } else { - errors = append(errors, fmt.Sprintf("JSON %v missing key %q", headerKey, name)) - } - } - } - } else { - errors = append(errors, fmt.Sprintf("Expected header in JSON %q but it was nil", headerKey)) - } - } else { - panic(fmt.Sprintf("Expected map[string]interface{} in json, got %T", json)) - } - - return errors -} - func testRestGet(url string) (*httptest.ResponseRecorder, error) { req, err := http.NewRequest("GET", url, nil) req.Header.Add("Accept", "application/json") @@ -184,3 +44,89 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { return buf } + +func decodedNumberEquals(t *testing.T, json interface{}, path string, want float64) { + t.Helper() + els := strings.Split(path, "/") + val, msg := getDecodedPath(json, els...) + if msg != "" { + t.Errorf("JSON result%s", msg) + return + } + if got, ok := val.(float64); ok { + if got == want { + return + } + } + t.Errorf("JSON result/%s == %v (%T), want: %v", path, val, val, want) +} + +func decodedStringEquals(t *testing.T, json interface{}, path string, want string) { + t.Helper() + els := strings.Split(path, "/") + val, msg := getDecodedPath(json, els...) + if msg != "" { + t.Errorf("JSON result%s", msg) + return + } + if got, ok := val.(string); ok { + if got == want { + return + } + } + t.Errorf("JSON result/%s == %v (%T), want: %v", path, val, val, want) +} + +// getDecodedPath recursively navigates the specified path, returing the requested element. If +// something goes wrong, the returned string will contain an explanation. +// +// Named path elements require the parent element to be a map[string]interface{}, numbers in square +// brackets require the parent element to be a []interface{}. +// +// getDecodedPath(o, "users", "[1]", "name") +// +// is equivalent to the JavaScript: +// +// o.users[1].name +// +func getDecodedPath(o interface{}, path ...string) (interface{}, string) { + if len(path) == 0 { + return o, "" + } + if o == nil { + return nil, " is nil" + } + key := path[0] + present := false + var val interface{} + if key[0] == '[' { + // Expecting slice. + index, err := strconv.Atoi(strings.Trim(key, "[]")) + if err != nil { + return nil, "/" + key + " is not a slice index" + } + oslice, ok := o.([]interface{}) + if !ok { + return nil, " is not a slice" + } + if index >= len(oslice) { + return nil, "/" + key + " is out of bounds" + } + val, present = oslice[index], true + } else { + // Expecting map. + omap, ok := o.(map[string]interface{}) + if !ok { + return nil, " is not a map" + } + val, present = omap[key] + } + if !present { + return nil, "/" + key + " is missing" + } + result, msg := getDecodedPath(val, path[1:]...) + if msg != "" { + return nil, "/" + key + msg + } + return result, "" +} From 0d0e07da70cd15182cc94e44f2fa6e97aad31cc5 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 13:58:47 -0700 Subject: [PATCH 28/82] file: Remove index and dir mutexes HashLock makes these redundant. #77 --- pkg/storage/file/fmessage.go | 7 +------ pkg/storage/file/fstore.go | 10 ---------- pkg/storage/file/mbox.go | 11 +---------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index cf375b4..79d1a65 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -1,7 +1,6 @@ package file import ( - "bufio" "io" "net/mail" "os" @@ -22,10 +21,6 @@ type Message struct { Fto []*mail.Address Fsubject string Fsize int64 - // These are for creating new messages only - writable bool - writerFile *os.File - writer *bufio.Writer } // newMessage creates a new FileMessage object and sets the Date and ID fields. @@ -48,7 +43,7 @@ func (mb *mbox) newMessage() (*Message, error) { } date := time.Now() id := generateID(date) - return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil + return &Message{mailbox: mb, Fid: id, Fdate: date}, nil } // Mailbox returns the name of the mailbox this message resides in. diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index ebe5c7e..57a26bf 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "time" "github.com/jhillyerd/inbucket/pkg/config" @@ -21,15 +20,6 @@ import ( const indexFileName = "index.gob" var ( - // indexMx is locked while reading/writing an index file - // - // NOTE: This is a bottleneck because it's a single lock even if we have a - // million index files - indexMx = new(sync.RWMutex) - - // dirMx is locked while creating/removing directories - dirMx = new(sync.Mutex) - // countChannel is filled with a sequential numbers (0000..9999), which are // used by generateID() to generate unique message IDs. It's global // because we only want one regardless of the number of DataStore objects diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index cb5c1b9..9a59984 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -101,9 +101,6 @@ func (mb *mbox) purge() error { func (mb *mbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] - // Lock for reading - indexMx.RLock() - defer indexMx.RUnlock() // Check if index exists if _, err := os.Stat(mb.indexPath); err != nil { // Does not exist, but that's not an error in our world @@ -146,8 +143,6 @@ func (mb *mbox) readIndex() error { // writeIndex overwrites the index on disk with the current mailbox data func (mb *mbox) writeIndex() error { // Lock for writing - indexMx.Lock() - defer indexMx.Unlock() if len(mb.messages) > 0 { // Ensure mailbox directory exists if err := mb.createDir(); err != nil { @@ -189,8 +184,6 @@ func (mb *mbox) writeIndex() error { // createDir checks for the presence of the path for this mailbox, creates it if needed func (mb *mbox) createDir() error { - dirMx.Lock() - defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { if err := os.MkdirAll(mb.path, 0770); err != nil { log.Errorf("Failed to create directory %v, %v", mb.path, err) @@ -202,8 +195,6 @@ func (mb *mbox) createDir() error { // removeDir removes the mailbox, plus empty higher level directories func (mb *mbox) removeDir() error { - dirMx.Lock() - defer dirMx.Unlock() // remove mailbox dir, including index file if err := os.RemoveAll(mb.path); err != nil { return err @@ -217,7 +208,7 @@ func (mb *mbox) removeDir() error { } // removeDirIfEmpty will remove the specified directory if it contains no files or directories. -// Caller should hold dirMx. Returns true if dir was removed. +// Returns true if dir was removed. func removeDirIfEmpty(path string) (removed bool) { f, err := os.Open(path) if err != nil { From 30f5c163e4d4687b36c56ca0cb9ecdade62f69c7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 14:30:56 -0700 Subject: [PATCH 29/82] log: Add locking to prevent race --- pkg/log/logging.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/log/logging.go b/pkg/log/logging.go index 7a73e24..7c06a1e 100644 --- a/pkg/log/logging.go +++ b/pkg/log/logging.go @@ -5,6 +5,7 @@ import ( golog "log" "os" "strings" + "sync" ) // Level is used to indicate the severity of a log entry @@ -30,12 +31,16 @@ var ( // logf is the file we send log output to, will be nil for stderr or stdout logf *os.File + + mu sync.RWMutex ) // Initialize logging. If logfile is equal to "stderr" or "stdout", then // we will log to that output stream. Otherwise the specificed file will // opened for writing, and all log data will be placed in it. func Initialize(logfile string) error { + mu.Lock() + defer mu.Unlock() if logfile != "stderr" { // stderr is the go logging default if logfile == "stdout" { @@ -55,6 +60,8 @@ func Initialize(logfile string) error { // SetLogLevel sets MaxLevel based on the provided string func SetLogLevel(level string) (ok bool) { + mu.Lock() + defer mu.Unlock() switch strings.ToUpper(level) { case "ERROR": MaxLevel = ERROR @@ -73,12 +80,16 @@ func SetLogLevel(level string) (ok bool) { // Errorf logs a message to the 'standard' Logger (always), accepts format strings func Errorf(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() msg = "[ERROR] " + msg golog.Printf(msg, args...) } // Warnf logs a message to the 'standard' Logger if MaxLevel is >= WARN, accepts format strings func Warnf(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= WARN { msg = "[WARN ] " + msg golog.Printf(msg, args...) @@ -87,6 +98,8 @@ func Warnf(msg string, args ...interface{}) { // Infof logs a message to the 'standard' Logger if MaxLevel is >= INFO, accepts format strings func Infof(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= INFO { msg = "[INFO ] " + msg golog.Printf(msg, args...) @@ -95,6 +108,8 @@ func Infof(msg string, args ...interface{}) { // Tracef logs a message to the 'standard' Logger if MaxLevel is >= TRACE, accepts format strings func Tracef(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= TRACE { msg = "[TRACE] " + msg golog.Printf(msg, args...) @@ -105,6 +120,8 @@ func Tracef(msg string, args ...interface{}) { // log rotation system the opportunity to move the existing log file out of the // way and have Inbucket create a new one. func Rotate() { + mu.Lock() + defer mu.Unlock() // Rotate logs if configured if logf != nil { closeLogFile() @@ -117,6 +134,8 @@ func Rotate() { // Close the log file if we have one open func Close() { + mu.Lock() + defer mu.Unlock() if logf != nil { closeLogFile() } From e5785e81aa4cafc7aceb77eaf955b3e0a1c3a2c8 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 15:14:48 -0700 Subject: [PATCH 30/82] Update CHANGELOG for refactor --- CHANGELOG.md | 6 ++++++ pkg/message/manager.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7874198..e674d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] + +### Changed +- Massive refactor of back-end code. Inbucket should now be both easier and + more enjoyable to work on. + ## [v1.3.1] - 2018-03-10 ### Fixed diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 2a8bb47..9ff4d66 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -45,7 +45,8 @@ func (s *StoreManager) Deliver( prefix string, source []byte, ) (string, error) { - // TODO enmime is too heavy for this step, only need header + // TODO enmime is too heavy for this step, only need header. + // Go's header parsing isn't good enough, so this is blocked on enmime issue #64. env, err := enmime.ReadEnvelope(bytes.NewReader(source)) if err != nil { return "", err From e7a86bd8f8ed62595433ba0ed6b4b5260b621654 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 20 Mar 2018 17:55:43 -0700 Subject: [PATCH 31/82] Hide envelope, use Part.Content for #85 --- pkg/message/manager.go | 2 +- pkg/message/message.go | 36 ++++++++++++++++++++++++++++++- pkg/rest/apiv1_controller.go | 28 ++++++++++++------------ pkg/rest/apiv1_controller_test.go | 8 +++---- pkg/webui/mailbox_controller.go | 27 ++++++++++------------- 5 files changed, 65 insertions(+), 36 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 9ff4d66..5e428e6 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -121,7 +121,7 @@ func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { } _ = r.Close() header := makeMetadata(sm) - return &Message{Metadata: *header, Envelope: env}, nil + return &Message{Metadata: *header, env: env}, nil } // PurgeMessages removes all messages from the specified mailbox. diff --git a/pkg/message/message.go b/pkg/message/message.go index 7f8bec0..8bdd460 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "net/mail" + "net/textproto" "time" "github.com/jhillyerd/enmime" @@ -25,7 +26,40 @@ type Metadata struct { // Message holds both the metadata and content of a message. type Message struct { Metadata - Envelope *enmime.Envelope + env *enmime.Envelope +} + +// New constructs a new Message +func New(m Metadata, e *enmime.Envelope) *Message { + return &Message{ + Metadata: m, + env: e, + } +} + +// Attachments returns the MIME attachments for the message. +func (m *Message) Attachments() []*enmime.Part { + return m.env.Attachments +} + +// Header returns the header map for this message. +func (m *Message) Header() textproto.MIMEHeader { + return m.env.Root.Header +} + +// HTML returns the HTML body of the message. +func (m *Message) HTML() string { + return m.env.HTML +} + +// MIMEErrors returns MIME parsing errors and warnings. +func (m *Message) MIMEErrors() []*enmime.Error { + return m.env.Errors +} + +// Text returns the plain text body of the message. +func (m *Message) Text() string { + return m.env.Text } // Delivery is used to add a message to storage. diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index c463ecf..34dea29 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -7,7 +7,6 @@ import ( "crypto/md5" "encoding/hex" - "io/ioutil" "strconv" "github.com/jhillyerd/inbucket/pkg/log" @@ -63,19 +62,20 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime := msg.Envelope - attachments := make([]*model.JSONMessageAttachmentV1, len(mime.Attachments)) - for i, att := range mime.Attachments { - var content []byte - content, err = ioutil.ReadAll(att) + attachParts := msg.Attachments() + attachments := make([]*model.JSONMessageAttachmentV1, len(attachParts)) + for i, part := range attachParts { + content := part.Content var checksum = md5.Sum(content) attachments[i] = &model.JSONMessageAttachmentV1{ - ContentType: att.ContentType, - FileName: att.FileName, - DownloadLink: "http://" + req.Host + "/mailbox/dattach/" + name + "/" + id + "/" + strconv.Itoa(i) + "/" + att.FileName, - ViewLink: "http://" + req.Host + "/mailbox/vattach/" + name + "/" + id + "/" + strconv.Itoa(i) + "/" + att.FileName, - MD5: hex.EncodeToString(checksum[:]), + ContentType: part.ContentType, + FileName: part.FileName, + DownloadLink: "http://" + req.Host + "/mailbox/dattach/" + name + "/" + id + "/" + + strconv.Itoa(i) + "/" + part.FileName, + ViewLink: "http://" + req.Host + "/mailbox/vattach/" + name + "/" + id + "/" + + strconv.Itoa(i) + "/" + part.FileName, + MD5: hex.EncodeToString(checksum[:]), } } @@ -88,10 +88,10 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( Subject: msg.Subject, Date: msg.Date, Size: msg.Size, - Header: mime.Root.Header, + Header: msg.Header(), Body: &model.JSONMessageBodyV1{ - Text: mime.Text, - HTML: mime.HTML, + Text: msg.Text(), + HTML: msg.HTML(), }, Attachments: attachments, }) diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 4906d21..56fa403 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -172,8 +172,8 @@ func TestRestMessage(t *testing.T) { } // Test JSON message headers - msg1 := &message.Message{ - Metadata: message.Metadata{ + msg1 := message.New( + message.Metadata{ Mailbox: "good", ID: "0001", From: &mail.Address{Name: "", Address: "from1@host"}, @@ -181,7 +181,7 @@ func TestRestMessage(t *testing.T) { Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), }, - Envelope: &enmime.Envelope{ + &enmime.Envelope{ Text: "This is some text", HTML: "This is some HTML", Root: &enmime.Part{ @@ -191,7 +191,7 @@ func TestRestMessage(t *testing.T) { }, }, }, - } + ) mm.AddMessage("good", msg1) // Check return code diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index d876b3a..e835d78 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -102,12 +102,11 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime := msg.Envelope - body := template.HTML(web.TextToHTML(mime.Text)) - htmlAvailable := mime.HTML != "" + body := template.HTML(web.TextToHTML(msg.Text())) + htmlAvailable := msg.HTML() != "" var htmlBody template.HTML if htmlAvailable { - if str, err := sanitize.HTML(mime.HTML); err == nil { + if str, err := sanitize.HTML(msg.HTML()); err == nil { htmlBody = template.HTML(str) } else { log.Warnf("HTML sanitizer failed: %s", err) @@ -121,8 +120,8 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er "body": body, "htmlAvailable": htmlAvailable, "htmlBody": htmlBody, - "mimeErrors": mime.Errors, - "attachments": mime.Attachments, + "mimeErrors": msg.MIMEErrors(), + "attachments": msg.Attachments(), }) } @@ -143,14 +142,13 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime := msg.Envelope // Render partial template w.Header().Set("Content-Type", "text/html; charset=UTF-8") return web.RenderPartial("mailbox/_html.html", w, map[string]interface{}{ "ctx": ctx, "name": name, "message": msg, - "body": template.HTML(mime.HTML), + "body": template.HTML(msg.HTML()), }) } @@ -206,18 +204,16 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body := msg.Envelope - if int(num) >= len(body.Attachments) { + if int(num) >= len(msg.Attachments()) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - part := body.Attachments[num] // Output attachment w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", "attachment") - _, err = io.Copy(w, part) + _, err = w.Write(msg.Attachments()[num].Content) return err } @@ -249,16 +245,15 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body := msg.Envelope - if int(num) >= len(body.Attachments) { + if int(num) >= len(msg.Attachments()) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - part := body.Attachments[num] // Output attachment + part := msg.Attachments()[num] w.Header().Set("Content-Type", part.ContentType) - _, err = io.Copy(w, part) + _, err = w.Write(part.Content) return err } From be940dd2bcdfa87f2411c9b991e0e3697cc85a65 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 20 Mar 2018 19:18:07 -0700 Subject: [PATCH 32/82] rest: fix timezone in controller tests --- pkg/rest/apiv1_controller_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 56fa403..734879e 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -67,13 +67,15 @@ func TestRestMailboxList(t *testing.T) { } // Test JSON message headers + tzPDT := time.FixedZone("PDT", -7*3600) + tzPST := time.FixedZone("PST", -8*3600) meta1 := message.Metadata{ Mailbox: "good", ID: "0001", From: &mail.Address{Name: "", Address: "from1@host"}, To: []*mail.Address{{Name: "", Address: "to1@host"}}, Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, tzPST), } meta2 := message.Metadata{ Mailbox: "good", @@ -81,7 +83,7 @@ func TestRestMailboxList(t *testing.T) { From: &mail.Address{Name: "", Address: "from2@host"}, To: []*mail.Address{{Name: "", Address: "to1@host"}}, Subject: "subject 2", - Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), + Date: time.Date(2012, 7, 1, 10, 11, 12, 253, tzPDT), } mm.AddMessage("good", &message.Message{Metadata: meta1}) mm.AddMessage("good", &message.Message{Metadata: meta2}) @@ -111,14 +113,14 @@ func TestRestMailboxList(t *testing.T) { decodedStringEquals(t, result, "[0]/from", "") decodedStringEquals(t, result, "[0]/to/[0]", "") decodedStringEquals(t, result, "[0]/subject", "subject 1") - decodedStringEquals(t, result, "[0]/date", "2012-02-01T10:11:12.000000253-00:13") + decodedStringEquals(t, result, "[0]/date", "2012-02-01T10:11:12.000000253-08:00") decodedNumberEquals(t, result, "[0]/size", 0) decodedStringEquals(t, result, "[1]/mailbox", "good") decodedStringEquals(t, result, "[1]/id", "0002") decodedStringEquals(t, result, "[1]/from", "") decodedStringEquals(t, result, "[1]/to/[0]", "") decodedStringEquals(t, result, "[1]/subject", "subject 2") - decodedStringEquals(t, result, "[1]/date", "2012-07-01T10:11:12.000000253-00:11") + decodedStringEquals(t, result, "[1]/date", "2012-07-01T10:11:12.000000253-07:00") decodedNumberEquals(t, result, "[1]/size", 0) if t.Failed() { @@ -172,6 +174,7 @@ func TestRestMessage(t *testing.T) { } // Test JSON message headers + tzPST := time.FixedZone("PST", -8*3600) msg1 := message.New( message.Metadata{ Mailbox: "good", @@ -179,7 +182,7 @@ func TestRestMessage(t *testing.T) { From: &mail.Address{Name: "", Address: "from1@host"}, To: []*mail.Address{{Name: "", Address: "to1@host"}}, Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, tzPST), }, &enmime.Envelope{ Text: "This is some text", @@ -216,7 +219,7 @@ func TestRestMessage(t *testing.T) { decodedStringEquals(t, result, "from", "") decodedStringEquals(t, result, "to/[0]", "") decodedStringEquals(t, result, "subject", "subject 1") - decodedStringEquals(t, result, "date", "2012-02-01T10:11:12.000000253-00:13") + decodedStringEquals(t, result, "date", "2012-02-01T10:11:12.000000253-08:00") decodedNumberEquals(t, result, "size", 0) decodedStringEquals(t, result, "body/text", "This is some text") decodedStringEquals(t, result, "body/html", "This is some HTML") From 845cbedc0dc52e63310618754afedd40e528f00a Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 21 Mar 2018 20:44:47 -0700 Subject: [PATCH 33/82] config: Replace robfig with envconfig for #86 - Initial envconfig system is working, not bulletproof. - Added sane defaults for required parameters. --- cmd/inbucket/main.go | 111 +++---- pkg/config/config.go | 304 ++++---------------- pkg/log/logging.go | 4 +- pkg/policy/address.go | 2 +- pkg/policy/address_test.go | 4 +- pkg/rest/testutils_test.go | 8 +- pkg/server/pop3/handler.go | 2 +- pkg/server/pop3/listener.go | 9 +- pkg/server/smtp/handler.go | 4 +- pkg/server/smtp/handler_test.go | 7 +- pkg/server/smtp/listener.go | 9 +- pkg/server/web/context.go | 27 +- pkg/server/web/server.go | 24 +- pkg/server/web/template.go | 6 +- pkg/storage/file/fstore.go | 2 +- pkg/storage/file/fstore_test.go | 14 +- pkg/storage/retention.go | 8 +- pkg/storage/retention_test.go | 6 +- pkg/webui/root_controller.go | 30 +- themes/bootstrap/templates/root/status.html | 8 +- 20 files changed, 190 insertions(+), 399 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 0de0b07..0183998 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -27,83 +27,63 @@ import ( ) var ( - // version contains the build version number, populated during linking + // version contains the build version number, populated during linking. version = "undefined" - // date contains the build date, populated during linking + // date contains the build date, populated during linking. date = "undefined" - - // Command line flags - help = flag.Bool("help", false, "Displays this help") - pidfile = flag.String("pidfile", "none", "Write our PID into the specified file") - logfile = flag.String("logfile", "stderr", "Write out log into the specified file") - - // shutdownChan - close it to tell Inbucket to shut down cleanly - shutdownChan = make(chan bool) - - // Server instances - smtpServer *smtp.Server - pop3Server *pop3.Server ) func init() { - flag.Usage = func() { - fmt.Fprintln(os.Stderr, "Usage of inbucket [options] :") - flag.PrintDefaults() - } - - // Server uptime for status page + // Server uptime for status page. startTime := time.Now() expvar.Publish("uptime", expvar.Func(func() interface{} { return time.Since(startTime) / time.Second })) - // Goroutine count for status page + // Goroutine count for status page. expvar.Publish("goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() })) } func main() { - config.Version = version - config.BuildDate = date - + // Command line flags. + help := flag.Bool("help", false, "Displays this help") + pidfile := flag.String("pidfile", "", "Write our PID into the specified file") + logfile := flag.String("logfile", "stderr", "Write out log into the specified file") + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "Usage: inbucket [options]") + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, "") + config.Usage() + } flag.Parse() if *help { flag.Usage() return } - - // Root context - rootCtx, rootCancel := context.WithCancel(context.Background()) - - // Load & Parse config - if flag.NArg() != 1 { - flag.Usage() - os.Exit(1) - } - err := config.LoadConfig(flag.Arg(0)) + // Process configuration. + config.Version = version + config.BuildDate = date + conf, err := config.Process() if err != nil { - fmt.Fprintf(os.Stderr, "Failed to parse config: %v\n", err) + fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err) os.Exit(1) } - - // Setup signal handler + // Setup signal handler. sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) - - // Initialize logging - log.SetLogLevel(config.GetLogLevel()) + // Initialize logging. + log.SetLogLevel(conf.LogLevel) if err := log.Initialize(*logfile); err != nil { fmt.Fprintf(os.Stderr, "%v", err) os.Exit(1) } defer log.Close() - log.Infof("Inbucket %v (%v) starting...", config.Version, config.BuildDate) - - // Write pidfile if requested - if *pidfile != "none" { + // Write pidfile if requested. + if *pidfile != "" { pidf, err := os.Create(*pidfile) if err != nil { log.Errorf("Failed to create %q: %v", *pidfile, err) @@ -114,26 +94,26 @@ func main() { log.Errorf("Failed to close PID file %q: %v", *pidfile, err) } } - // Configure internal services. - msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) - dscfg := config.GetDataStoreConfig() - store := file.New(dscfg) - apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} + rootCtx, rootCancel := context.WithCancel(context.Background()) + shutdownChan := make(chan bool) + msgHub := msghub.New(rootCtx, conf.Web.MonitorHistory) + store := file.New(conf.Storage) + addrPolicy := &policy.Addressing{Config: conf.SMTP} mmanager := &message.StoreManager{Store: store, Hub: msgHub} // Start Retention scanner. - retentionScanner := storage.NewRetentionScanner(dscfg, store, shutdownChan) + retentionScanner := storage.NewRetentionScanner(conf.Storage, store, shutdownChan) retentionScanner.Start() // Start HTTP server. - web.Initialize(config.GetWebConfig(), shutdownChan, mmanager, msgHub) + web.Initialize(conf, shutdownChan, mmanager, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) // Start POP3 server. - pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, store) + pop3Server := pop3.New(conf.POP3, shutdownChan, store) go pop3Server.Start(rootCtx) // Start SMTP server. - smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, mmanager, apolicy) + smtpServer := smtp.NewServer(conf.SMTP, shutdownChan, mmanager, addrPolicy) go smtpServer.Start(rootCtx) // Loop forever waiting for signals or shutdown channel. signalLoop: @@ -158,30 +138,27 @@ signalLoop: break signalLoop } } - - // Wait for active connections to finish - go timedExit() + // Wait for active connections to finish. + go timedExit(*pidfile) smtpServer.Drain() pop3Server.Drain() retentionScanner.Join() - - removePIDFile() + removePIDFile(*pidfile) } -// removePIDFile removes the PID file if created -func removePIDFile() { - if *pidfile != "none" { - if err := os.Remove(*pidfile); err != nil { - log.Errorf("Failed to remove %q: %v", *pidfile, err) +// removePIDFile removes the PID file if created. +func removePIDFile(pidfile string) { + if pidfile != "" { + if err := os.Remove(pidfile); err != nil { + log.Errorf("Failed to remove %q: %v", pidfile, err) } } } -// timedExit is called as a goroutine during shutdown, it will force an exit -// after 15 seconds -func timedExit() { +// timedExit is called as a goroutine during shutdown, it will force an exit after 15 seconds. +func timedExit(pidfile string) { time.Sleep(15 * time.Second) log.Errorf("Clean shutdown took too long, forcing exit") - removePIDFile() + removePIDFile(pidfile) os.Exit(0) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 01e909e..11ef51e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,61 +1,22 @@ package config import ( - "fmt" - "net" + "log" "os" - "sort" - "strings" + "text/tabwriter" + "time" - "github.com/robfig/config" + "github.com/kelseyhightower/envconfig" ) -// SMTPConfig contains the SMTP server configuration - not using pointers so that we can pass around -// copies of the object safely. -type SMTPConfig struct { - IP4address net.IP - IP4port int - Domain string - DomainNoStore string - MaxRecipients int - MaxIdleSeconds int - MaxMessageBytes int - StoreMessages bool -} - -// POP3Config contains the POP3 server configuration -type POP3Config struct { - IP4address net.IP - IP4port int - Domain string - MaxIdleSeconds int -} - -// WebConfig contains the HTTP server configuration -type WebConfig struct { - IP4address net.IP - IP4port int - TemplateDir string - TemplateCache bool - PublicDir string - GreetingFile string - MailboxPrompt string - CookieAuthKey string - MonitorVisible bool - MonitorHistory int -} - -// DataStoreConfig contains the mail store configuration -type DataStoreConfig struct { - Path string - RetentionMinutes int - RetentionSleep int - MailboxMsgCap int -} - const ( - missingErrorFmt = "[%v] missing required option %q" - parseErrorFmt = "[%v] option %q error: %v" + prefix = "inbucket" + tableFormat = `Inbucket is configured via the environment. The following environment +variables can be used: + +KEY DEFAULT REQUIRED DESCRIPTION +{{range .}}{{usage_key .}} {{usage_default .}} {{usage_required .}} {{usage_description .}} +{{end}}` ) var ( @@ -64,207 +25,68 @@ var ( // BuildDate for this build, set by main BuildDate = "" - - // Config is our global robfig/config object - Config *config.Config - logLevel string - - // Parsed specific configs - smtpConfig = &SMTPConfig{} - pop3Config = &POP3Config{} - webConfig = &WebConfig{} - dataStoreConfig = &DataStoreConfig{} ) -// GetSMTPConfig returns a copy of the SmtpConfig object -func GetSMTPConfig() SMTPConfig { - return *smtpConfig +// Root wraps all other configurations. +type Root struct { + LogLevel string `required:"true" default:"INFO" desc:"TRACE, INFO, WARN, or ERROR"` + SMTP SMTP + POP3 POP3 + Web Web + Storage Storage } -// GetPOP3Config returns a copy of the Pop3Config object -func GetPOP3Config() POP3Config { - return *pop3Config +// SMTP contains the SMTP server configuration. +type SMTP struct { + Addr string `required:"true" default:"0.0.0.0:2500" desc:"SMTP server IP4 host:port"` + Domain string `required:"true" default:"inbucket" desc:"HELO domain"` + DomainNoStore string `desc:"Load testing domain"` + MaxRecipients int `required:"true" default:"200" desc:"Maximum RCPT TO per message"` + MaxIdle time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` + MaxMessageBytes int `required:"true" default:"2048000" desc:"Maximum message size"` + StoreMessages bool `required:"true" default:"true" desc:"Store incoming mail?"` } -// GetWebConfig returns a copy of the WebConfig object -func GetWebConfig() WebConfig { - return *webConfig +// POP3 contains the POP3 server configuration. +type POP3 struct { + Addr string `required:"true" default:"0.0.0.0:1100" desc:"POP3 server IP4 host:port"` + Domain string `required:"true" default:"inbucket" desc:"HELLO domain"` + MaxIdle time.Duration `required:"true" default:"600s" desc:"Idle network timeout"` } -// GetDataStoreConfig returns a copy of the DataStoreConfig object -func GetDataStoreConfig() DataStoreConfig { - return *dataStoreConfig +// Web contains the HTTP server configuration. +type Web struct { + Addr string `required:"true" default:"0.0.0.0:9000" desc:"Web server IP4 host:port"` + TemplateDir string `required:"true" default:"themes/bootstrap/templates" desc:"Theme template dir"` + TemplateCache bool `required:"true" default:"true" desc:"Cache templates after first use?"` + PublicDir string `required:"true" default:"themes/bootstrap/public" desc:"Theme public dir"` + GreetingFile string `required:"true" default:"themes/greeting.html" desc:"Home page greeting HTML"` + MailboxPrompt string `required:"true" default:"@inbucket" desc:"Prompt next to mailbox input"` + CookieAuthKey string `desc:"Session cipher key (text)"` + MonitorVisible bool `required:"true" default:"true" desc:"Show monitor tab in UI?"` + MonitorHistory int `required:"true" default:"30" desc:"Monitor remembered messages"` } -// GetLogLevel returns the configured log level -func GetLogLevel() string { - return logLevel +// Storage contains the mail store configuration. +type Storage struct { + Path string `required:"true" default:"/tmp/inbucket" desc:"Mail store path"` + RetentionPeriod time.Duration `required:"true" default:"24h" desc:"Duration to retain messages"` + RetentionSleep time.Duration `required:"true" default:"100ms" desc:"Duration to sleep between deletes"` + MailboxMsgCap int `required:"true" default:"500" desc:"Maximum messages per mailbox"` } -// LoadConfig loads the specified configuration file into inbucket.Config and performs validations -// on it. -func LoadConfig(filename string) error { - var err error - Config, err = config.ReadDefault(filename) - if err != nil { - return err - } - // Validation error messages - messages := make([]string, 0) - // Validate sections - for _, s := range []string{"logging", "smtp", "pop3", "web", "datastore"} { - if !Config.HasSection(s) { - messages = append(messages, - fmt.Sprintf("Config section [%v] is required", s)) - } - } - // Return immediately if config is missing entire sections - if len(messages) > 0 { - fmt.Fprintln(os.Stderr, "Error(s) validating configuration:") - for _, m := range messages { - fmt.Fprintln(os.Stderr, " -", m) - } - return fmt.Errorf("Failed to validate configuration") - } - // Load string config options - stringOptions := []struct { - section string - name string - target *string - required bool - }{ - {"logging", "level", &logLevel, true}, - {"smtp", "domain", &smtpConfig.Domain, true}, - {"smtp", "domain.nostore", &smtpConfig.DomainNoStore, false}, - {"pop3", "domain", &pop3Config.Domain, true}, - {"web", "template.dir", &webConfig.TemplateDir, true}, - {"web", "public.dir", &webConfig.PublicDir, true}, - {"web", "greeting.file", &webConfig.GreetingFile, true}, - {"web", "mailbox.prompt", &webConfig.MailboxPrompt, false}, - {"web", "cookie.auth.key", &webConfig.CookieAuthKey, false}, - {"datastore", "path", &dataStoreConfig.Path, true}, - } - for _, opt := range stringOptions { - str, err := Config.String(opt.section, opt.name) - if Config.HasOption(opt.section, opt.name) && err != nil { - messages = append(messages, fmt.Sprintf(parseErrorFmt, opt.section, opt.name, err)) - continue - } - if str == "" && opt.required { - messages = append(messages, fmt.Sprintf(missingErrorFmt, opt.section, opt.name)) - } - *opt.target = str - } - // Load boolean config options - boolOptions := []struct { - section string - name string - target *bool - required bool - }{ - {"smtp", "store.messages", &smtpConfig.StoreMessages, true}, - {"web", "template.cache", &webConfig.TemplateCache, true}, - {"web", "monitor.visible", &webConfig.MonitorVisible, true}, - } - for _, opt := range boolOptions { - if Config.HasOption(opt.section, opt.name) { - flag, err := Config.Bool(opt.section, opt.name) - if err != nil { - messages = append(messages, fmt.Sprintf(parseErrorFmt, opt.section, opt.name, err)) - } - *opt.target = flag - } else { - if opt.required { - messages = append(messages, fmt.Sprintf(missingErrorFmt, opt.section, opt.name)) - } - } - } - // Load integer config options - intOptions := []struct { - section string - name string - target *int - required bool - }{ - {"smtp", "ip4.port", &smtpConfig.IP4port, true}, - {"smtp", "max.recipients", &smtpConfig.MaxRecipients, true}, - {"smtp", "max.idle.seconds", &smtpConfig.MaxIdleSeconds, true}, - {"smtp", "max.message.bytes", &smtpConfig.MaxMessageBytes, true}, - {"pop3", "ip4.port", &pop3Config.IP4port, true}, - {"pop3", "max.idle.seconds", &pop3Config.MaxIdleSeconds, true}, - {"web", "ip4.port", &webConfig.IP4port, true}, - {"web", "monitor.history", &webConfig.MonitorHistory, true}, - {"datastore", "retention.minutes", &dataStoreConfig.RetentionMinutes, true}, - {"datastore", "retention.sleep.millis", &dataStoreConfig.RetentionSleep, true}, - {"datastore", "mailbox.message.cap", &dataStoreConfig.MailboxMsgCap, true}, - } - for _, opt := range intOptions { - if Config.HasOption(opt.section, opt.name) { - num, err := Config.Int(opt.section, opt.name) - if err != nil { - messages = append(messages, fmt.Sprintf(parseErrorFmt, opt.section, opt.name, err)) - } - *opt.target = num - } else { - if opt.required { - messages = append(messages, fmt.Sprintf(missingErrorFmt, opt.section, opt.name)) - } - } - } - // Load IP address config options - ipOptions := []struct { - section string - name string - target *net.IP - required bool - }{ - {"smtp", "ip4.address", &smtpConfig.IP4address, true}, - {"pop3", "ip4.address", &pop3Config.IP4address, true}, - {"web", "ip4.address", &webConfig.IP4address, true}, - } - for _, opt := range ipOptions { - if Config.HasOption(opt.section, opt.name) { - str, err := Config.String(opt.section, opt.name) - if err != nil { - messages = append(messages, fmt.Sprintf(parseErrorFmt, opt.section, opt.name, err)) - continue - } - addr := net.ParseIP(str) - if addr == nil { - messages = append(messages, - fmt.Sprintf("Failed to parse IP [%v]%v: %q", opt.section, opt.name, str)) - continue - } - addr = addr.To4() - if addr == nil { - messages = append(messages, - fmt.Sprintf("Failed to parse IP [%v]%v: %q not IPv4!", - opt.section, opt.name, str)) - } - *opt.target = addr - } else { - if opt.required { - messages = append(messages, fmt.Sprintf(missingErrorFmt, opt.section, opt.name)) - } - } - } - // Validate log level - switch strings.ToUpper(logLevel) { - case "": - // Missing was already reported - case "TRACE", "INFO", "WARN", "ERROR": - default: - messages = append(messages, - fmt.Sprintf("Invalid value provided for [logging]level: %q", logLevel)) - } - // Print messages and return error if any validations failed - if len(messages) > 0 { - fmt.Fprintln(os.Stderr, "Error(s) validating configuration:") - sort.Strings(messages) - for _, m := range messages { - fmt.Fprintln(os.Stderr, " -", m) - } - return fmt.Errorf("Failed to validate configuration") - } - return nil +// Process loads and parses configuration from the environment. +func Process() (*Root, error) { + c := &Root{} + err := envconfig.Process(prefix, c) + return c, err +} + +// Usage prints out the envconfig usage to Stderr. +func Usage() { + tabs := tabwriter.NewWriter(os.Stderr, 1, 0, 4, ' ', 0) + if err := envconfig.Usagef(prefix, &Root{}, tabs, tableFormat); err != nil { + log.Fatalf("Unable to parse env config: %v", err) + } + tabs.Flush() } diff --git a/pkg/log/logging.go b/pkg/log/logging.go index 7c06a1e..c2184ff 100644 --- a/pkg/log/logging.go +++ b/pkg/log/logging.go @@ -72,7 +72,7 @@ func SetLogLevel(level string) (ok bool) { case "TRACE": MaxLevel = TRACE default: - Errorf("Unknown log level requested: " + level) + golog.Print("Error, unknown log level requested: " + level) return false } return true @@ -150,7 +150,6 @@ func openLogFile() error { return fmt.Errorf("failed to create %v: %v", logfname, err) } golog.SetOutput(logf) - Tracef("Opened new logfile") // Platform specific reassignStdout() return nil @@ -158,7 +157,6 @@ func openLogFile() error { // closeLogFile closes the current logfile func closeLogFile() { - Tracef("Closing logfile") // We are never in a situation where we can do anything about failing to close _ = logf.Close() } diff --git a/pkg/policy/address.go b/pkg/policy/address.go index 34ed8f8..680a789 100644 --- a/pkg/policy/address.go +++ b/pkg/policy/address.go @@ -11,7 +11,7 @@ import ( // Addressing handles email address policy. type Addressing struct { - Config config.SMTPConfig + Config config.SMTP } // NewRecipient parses an address into a Recipient. diff --git a/pkg/policy/address_test.go b/pkg/policy/address_test.go index 2ab2c60..009033c 100644 --- a/pkg/policy/address_test.go +++ b/pkg/policy/address_test.go @@ -11,7 +11,7 @@ import ( func TestShouldStoreDomain(t *testing.T) { // Test with storage enabled. ap := &policy.Addressing{ - Config: config.SMTPConfig{ + Config: config.SMTP{ DomainNoStore: "Foo.Com", StoreMessages: true, }, @@ -36,7 +36,7 @@ func TestShouldStoreDomain(t *testing.T) { } // Test with storage disabled. ap = &policy.Addressing{ - Config: config.SMTPConfig{ + Config: config.SMTP{ StoreMessages: false, }, } diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 587c3b7..e2f2e3d 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -34,9 +34,11 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { // Have to reset default mux to prevent duplicate routes http.DefaultServeMux = http.NewServeMux() - cfg := config.WebConfig{ - TemplateDir: "../themes/bootstrap/templates", - PublicDir: "../themes/bootstrap/public", + cfg := &config.Root{ + Web: config.Web{ + TemplateDir: "../themes/bootstrap/templates", + PublicDir: "../themes/bootstrap/public", + }, } shutdownChan := make(chan bool) web.Initialize(cfg, shutdownChan, mm, &msghub.Hub{}) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index f8229ca..00334fa 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -536,7 +536,7 @@ func (ses *Session) enterState(state State) { // Calculate the next read or write deadline based on maxIdleSeconds func (ses *Session) nextDeadline() time.Time { - return time.Now().Add(time.Duration(ses.server.maxIdleSeconds) * time.Second) + return time.Now().Add(ses.server.maxIdle) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index c971854..ee6d2da 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -2,7 +2,6 @@ package pop3 import ( "context" - "fmt" "net" "sync" "time" @@ -16,7 +15,7 @@ import ( type Server struct { host string domain string - maxIdleSeconds int + maxIdle time.Duration dataStore storage.Store listener net.Listener globalShutdown chan bool @@ -24,12 +23,12 @@ type Server struct { } // New creates a new Server struct -func New(cfg config.POP3Config, shutdownChan chan bool, ds storage.Store) *Server { +func New(cfg config.POP3, shutdownChan chan bool, ds storage.Store) *Server { return &Server{ - host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), + host: cfg.Addr, domain: cfg.Domain, dataStore: ds, - maxIdleSeconds: cfg.MaxIdleSeconds, + maxIdle: cfg.MaxIdle, globalShutdown: shutdownChan, waitgroup: new(sync.WaitGroup), } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 4d6ee33..12338b2 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -412,9 +412,9 @@ func (ss *Session) greet() { ss.send(fmt.Sprintf("220 %v Inbucket SMTP ready", ss.server.domain)) } -// Calculate the next read or write deadline based on maxIdleSeconds +// Calculate the next read or write deadline based on maxIdle func (ss *Session) nextDeadline() time.Time { - return time.Now().Add(time.Duration(ss.server.maxIdleSeconds) * time.Second) + return time.Now().Add(ss.server.maxIdle) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 283c6d1..e047230 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -361,13 +361,12 @@ func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown func()) { // Test Server Config - cfg := config.SMTPConfig{ - IP4address: net.IPv4(127, 0, 0, 1), - IP4port: 2500, + cfg := config.SMTP{ + Addr: "127.0.0.1:2500", Domain: "inbucket.local", DomainNoStore: "bitbucket.local", MaxRecipients: 5, - MaxIdleSeconds: 5, + MaxIdle: 5, MaxMessageBytes: 5000, StoreMessages: true, } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 959b6f5..e56ac93 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -4,7 +4,6 @@ import ( "container/list" "context" "expvar" - "fmt" "net" "strings" "sync" @@ -43,7 +42,7 @@ type Server struct { domain string domainNoStore string maxRecips int - maxIdleSeconds int + maxIdle time.Duration maxMessageBytes int storeMessages bool @@ -80,17 +79,17 @@ var ( // NewServer creates a new Server instance with the specificed config func NewServer( - cfg config.SMTPConfig, + cfg config.SMTP, globalShutdown chan bool, manager message.Manager, apolicy *policy.Addressing, ) *Server { return &Server{ - host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), + host: cfg.Addr, domain: cfg.Domain, domainNoStore: strings.ToLower(cfg.DomainNoStore), maxRecips: cfg.MaxRecipients, - maxIdleSeconds: cfg.MaxIdleSeconds, + maxIdle: cfg.MaxIdle, maxMessageBytes: cfg.MaxMessageBytes, storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index 9ec2ab8..65e6b3d 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -12,13 +12,15 @@ import ( ) // Context is passed into every request handler function +// TODO remove redundant web config type Context struct { - Vars map[string]string - Session *sessions.Session - MsgHub *msghub.Hub - Manager message.Manager - WebConfig config.WebConfig - IsJSON bool + Vars map[string]string + Session *sessions.Session + MsgHub *msghub.Hub + Manager message.Manager + RootConfig *config.Root + WebConfig config.Web + IsJSON bool } // Close the Context (currently does nothing) @@ -57,12 +59,13 @@ func NewContext(req *http.Request) (*Context, error) { err = nil } ctx := &Context{ - Vars: vars, - Session: sess, - MsgHub: msgHub, - Manager: manager, - WebConfig: webConfig, - IsJSON: headerMatch(req, "Accept", "application/json"), + Vars: vars, + Session: sess, + MsgHub: msgHub, + Manager: manager, + RootConfig: rootConfig, + WebConfig: rootConfig.Web, + IsJSON: headerMatch(req, "Accept", "application/json"), } return ctx, err } diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index b3e8610..96716f0 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -4,7 +4,6 @@ package web import ( "context" "expvar" - "fmt" "net" "net/http" "time" @@ -30,7 +29,7 @@ var ( // incoming requests to the correct handler function Router = mux.NewRouter() - webConfig config.WebConfig + rootConfig *config.Root server *http.Server listener net.Listener sessionStore sessions.Store @@ -47,12 +46,12 @@ func init() { // Initialize sets up things for unit tests or the Start() method func Initialize( - cfg config.WebConfig, + conf *config.Root, shutdownChan chan bool, mm message.Manager, mh *msghub.Hub) { - webConfig = cfg + rootConfig = conf globalShutdown = shutdownChan // NewContext() will use this DataStore for the web handlers @@ -60,36 +59,35 @@ func Initialize( manager = mm // Content Paths - log.Infof("HTTP templates mapped to %q", cfg.TemplateDir) - log.Infof("HTTP static content mapped to %q", cfg.PublicDir) + log.Infof("HTTP templates mapped to %q", conf.Web.TemplateDir) + log.Infof("HTTP static content mapped to %q", conf.Web.PublicDir) Router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", - http.FileServer(http.Dir(cfg.PublicDir)))) + http.FileServer(http.Dir(conf.Web.PublicDir)))) http.Handle("/", Router) // Session cookie setup - if cfg.CookieAuthKey == "" { + if conf.Web.CookieAuthKey == "" { log.Infof("HTTP generating random cookie.auth.key") sessionStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64)) } else { log.Tracef("HTTP using configured cookie.auth.key") - sessionStore = sessions.NewCookieStore([]byte(cfg.CookieAuthKey)) + sessionStore = sessions.NewCookieStore([]byte(conf.Web.CookieAuthKey)) } } // Start begins listening for HTTP requests func Start(ctx context.Context) { - addr := fmt.Sprintf("%v:%v", webConfig.IP4address, webConfig.IP4port) server = &http.Server{ - Addr: addr, + Addr: rootConfig.Web.Addr, Handler: nil, ReadTimeout: 60 * time.Second, WriteTimeout: 60 * time.Second, } // We don't use ListenAndServe because it lacks a way to close the listener - log.Infof("HTTP listening on TCP4 %v", addr) + log.Infof("HTTP listening on TCP4 %v", server.Addr) var err error - listener, err = net.Listen("tcp", addr) + listener, err = net.Listen("tcp", server.Addr) if err != nil { log.Errorf("HTTP failed to start TCP4 listener: %v", err) emergencyShutdown() diff --git a/pkg/server/web/template.go b/pkg/server/web/template.go index 216a1a4..2597ada 100644 --- a/pkg/server/web/template.go +++ b/pkg/server/web/template.go @@ -50,7 +50,7 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { } tempPath := strings.Replace(name, "/", string(filepath.Separator), -1) - tempFile := filepath.Join(webConfig.TemplateDir, tempPath) + tempFile := filepath.Join(rootConfig.Web.TemplateDir, tempPath) log.Tracef("Parsing template %v", tempFile) var err error @@ -62,14 +62,14 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { t, err = t.ParseFiles(tempFile) } else { t = template.New("_base.html").Funcs(TemplateFuncs) - t, err = t.ParseFiles(filepath.Join(webConfig.TemplateDir, "_base.html"), tempFile) + t, err = t.ParseFiles(filepath.Join(rootConfig.Web.TemplateDir, "_base.html"), tempFile) } if err != nil { return nil, err } // Allows us to disable caching for theme development - if webConfig.TemplateCache { + if rootConfig.Web.TemplateCache { if partial { log.Tracef("Caching partial %v", name) cachedTemplates[name] = t diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 57a26bf..069a3fe 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -48,7 +48,7 @@ type Store struct { } // New creates a new DataStore object using the specified path -func New(cfg config.DataStoreConfig) storage.Store { +func New(cfg config.Storage) storage.Store { path := cfg.Path if path == "" { log.Errorf("No value configured for datastore path") diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 589bf18..0b42e86 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -23,7 +23,7 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { test.StoreSuite(t, func() (storage.Store, func(), error) { - ds, _ := setupDataStore(config.DataStoreConfig{}) + ds, _ := setupDataStore(config.Storage{}) destroy := func() { teardownDataStore(ds) } @@ -33,7 +33,7 @@ func TestSuite(t *testing.T) { // Test directory structure created by filestore func TestFSDirStructure(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) + ds, logbuf := setupDataStore(config.Storage{}) defer teardownDataStore(ds) root := ds.path @@ -111,7 +111,7 @@ func TestFSDirStructure(t *testing.T) { // Test missing files func TestFSMissing(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) + ds, logbuf := setupDataStore(config.Storage{}) defer teardownDataStore(ds) mbName := "fred" @@ -147,7 +147,7 @@ func TestFSMissing(t *testing.T) { // Test delivering several messages to the same mailbox, see if message cap works func TestFSMessageCap(t *testing.T) { mbCap := 10 - ds, logbuf := setupDataStore(config.DataStoreConfig{MailboxMsgCap: mbCap}) + ds, logbuf := setupDataStore(config.Storage{MailboxMsgCap: mbCap}) defer teardownDataStore(ds) mbName := "captain" @@ -188,7 +188,7 @@ func TestFSMessageCap(t *testing.T) { // Test delivering several messages to the same mailbox, see if no message cap works func TestFSNoMessageCap(t *testing.T) { mbCap := 0 - ds, logbuf := setupDataStore(config.DataStoreConfig{MailboxMsgCap: mbCap}) + ds, logbuf := setupDataStore(config.Storage{MailboxMsgCap: mbCap}) defer teardownDataStore(ds) mbName := "captain" @@ -218,7 +218,7 @@ func TestFSNoMessageCap(t *testing.T) { // Test Get the latest message func TestGetLatestMessage(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) + ds, logbuf := setupDataStore(config.Storage{}) defer teardownDataStore(ds) // james hashes to 474ba67bdb289c6263b36dfd8a7bed6c85b04943 @@ -260,7 +260,7 @@ func TestGetLatestMessage(t *testing.T) { } // setupDataStore creates a new FileDataStore in a temporary directory -func setupDataStore(cfg config.DataStoreConfig) (*Store, *bytes.Buffer) { +func setupDataStore(cfg config.Storage) (*Store, *bytes.Buffer) { path, err := ioutil.TempDir("", "inbucket") if err != nil { panic(err) diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 6c7adb0..d083152 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -54,7 +54,7 @@ type RetentionScanner struct { // NewRetentionScanner configures a new RententionScanner. func NewRetentionScanner( - cfg config.DataStoreConfig, + cfg config.Storage, ds Store, shutdownChannel chan bool, ) *RetentionScanner { @@ -62,11 +62,11 @@ func NewRetentionScanner( globalShutdown: shutdownChannel, retentionShutdown: make(chan bool), ds: ds, - retentionPeriod: time.Duration(cfg.RetentionMinutes) * time.Minute, - retentionSleep: time.Duration(cfg.RetentionSleep) * time.Millisecond, + retentionPeriod: cfg.RetentionPeriod, + retentionSleep: cfg.RetentionSleep, } // expRetentionPeriod is displayed on the status page - expRetentionPeriod.Set(int64(cfg.RetentionMinutes * 60)) + expRetentionPeriod.Set(int64(cfg.RetentionPeriod / time.Second)) return rs } diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index 234a377..e918cc1 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -27,9 +27,9 @@ func TestDoRetentionScan(t *testing.T) { ds.AddMessage(new2) ds.AddMessage(new3) // Test 4 hour retention - cfg := config.DataStoreConfig{ - RetentionMinutes: 239, - RetentionSleep: 0, + cfg := config.Storage{ + RetentionPeriod: 239 * time.Minute, + RetentionSleep: 0, } shutdownChan := make(chan bool) rs := storage.NewRetentionScanner(cfg, ds, shutdownChan) diff --git a/pkg/webui/root_controller.go b/pkg/webui/root_controller.go index 0ea5780..352d51b 100644 --- a/pkg/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -12,7 +12,7 @@ import ( // RootIndex serves the Inbucket landing page func RootIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - greeting, err := ioutil.ReadFile(config.GetWebConfig().GreetingFile) + greeting, err := ioutil.ReadFile(ctx.RootConfig.Web.GreetingFile) if err != nil { return fmt.Errorf("Failed to load greeting: %v", err) } @@ -31,7 +31,7 @@ func RootIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err // RootMonitor serves the Inbucket monitor page func RootMonitor(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - if !config.GetWebConfig().MonitorVisible { + if !ctx.RootConfig.Web.MonitorVisible { ctx.Session.AddFlash("Monitor is disabled in configuration", "errors") _ = ctx.Session.Save(req, w) http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) @@ -51,7 +51,7 @@ func RootMonitor(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // RootMonitorMailbox serves the Inbucket monitor page for a particular mailbox func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - if !config.GetWebConfig().MonitorVisible { + if !ctx.RootConfig.Web.MonitorVisible { ctx.Session.AddFlash("Monitor is disabled in configuration", "errors") _ = ctx.Session.Save(req, w) http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) @@ -79,12 +79,6 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Conte // RootStatus serves the Inbucket status page func RootStatus(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - smtpListener := fmt.Sprintf("%s:%d", config.GetSMTPConfig().IP4address.String(), - config.GetSMTPConfig().IP4port) - pop3Listener := fmt.Sprintf("%s:%d", config.GetPOP3Config().IP4address.String(), - config.GetPOP3Config().IP4port) - webListener := fmt.Sprintf("%s:%d", config.GetWebConfig().IP4address.String(), - config.GetWebConfig().IP4port) // Get flash messages, save session errorFlash := ctx.Session.Flashes("errors") if err = ctx.Session.Save(req, w); err != nil { @@ -92,14 +86,14 @@ func RootStatus(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err } // Render template return web.RenderTemplate("root/status.html", w, map[string]interface{}{ - "ctx": ctx, - "errorFlash": errorFlash, - "version": config.Version, - "buildDate": config.BuildDate, - "smtpListener": smtpListener, - "pop3Listener": pop3Listener, - "webListener": webListener, - "smtpConfig": config.GetSMTPConfig(), - "dataStoreConfig": config.GetDataStoreConfig(), + "ctx": ctx, + "errorFlash": errorFlash, + "version": config.Version, + "buildDate": config.BuildDate, + "smtpListener": ctx.RootConfig.SMTP.Addr, + "pop3Listener": ctx.RootConfig.POP3.Addr, + "webListener": ctx.RootConfig.Web.Addr, + "smtpConfig": ctx.RootConfig.SMTP, + "storageConfig": ctx.RootConfig.Storage, }) } diff --git a/themes/bootstrap/templates/root/status.html b/themes/bootstrap/templates/root/status.html index 0613e14..31fbc35 100644 --- a/themes/bootstrap/templates/root/status.html +++ b/themes/bootstrap/templates/root/status.html @@ -62,7 +62,7 @@ $(document).ready(
Message Cap:
- {{with .dataStoreConfig}} + {{with .storageConfig}} {{.MailboxMsgCap}} messages per mailbox {{end}}
@@ -167,14 +167,14 @@ $(document).ready(

- Data Store Metrics

+ Storage Metrics
Retention Period:
- {{if .dataStoreConfig.RetentionMinutes}} + {{if .storageConfig.RetentionPeriod}} . {{else}} Disabled @@ -184,7 +184,7 @@ $(document).ready(
Retention Scan:
- {{if .dataStoreConfig.RetentionMinutes}} + {{if .storageConfig.RetentionPeriod}} Completed . ago {{else}} Disabled From f0a94f48482319a1d8ddf6ff2b0bd62f98bd21ba Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Thu, 22 Mar 2018 20:02:34 -0700 Subject: [PATCH 34/82] More config cleanups for #86 --- cmd/inbucket/main.go | 10 +++++----- pkg/config/config.go | 12 ++++++------ pkg/server/pop3/handler.go | 6 +++--- pkg/server/pop3/listener.go | 10 +++++----- pkg/server/smtp/handler.go | 2 +- pkg/server/smtp/handler_test.go | 2 +- pkg/server/smtp/listener.go | 4 ++-- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 0183998..a54a8d8 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -49,18 +49,18 @@ func init() { func main() { // Command line flags. - help := flag.Bool("help", false, "Displays this help") - pidfile := flag.String("pidfile", "", "Write our PID into the specified file") - logfile := flag.String("logfile", "stderr", "Write out log into the specified file") + help := flag.Bool("help", false, "Displays help on flags and env variables.") + pidfile := flag.String("pidfile", "", "Write our PID into the specified file.") + logfile := flag.String("logfile", "stderr", "Write out log into the specified file.") flag.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: inbucket [options]") flag.PrintDefaults() - fmt.Fprintln(os.Stderr, "") - config.Usage() } flag.Parse() if *help { flag.Usage() + fmt.Fprintln(os.Stderr, "") + config.Usage() return } // Process configuration. diff --git a/pkg/config/config.go b/pkg/config/config.go index 11ef51e..d6b4b68 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -11,11 +11,11 @@ import ( const ( prefix = "inbucket" - tableFormat = `Inbucket is configured via the environment. The following environment -variables can be used: + tableFormat = `Inbucket is configured via the environment. The following environment variables +can be used: -KEY DEFAULT REQUIRED DESCRIPTION -{{range .}}{{usage_key .}} {{usage_default .}} {{usage_required .}} {{usage_description .}} +KEY DEFAULT DESCRIPTION +{{range .}}{{usage_key .}} {{usage_default .}} {{usage_description .}} {{end}}` ) @@ -42,16 +42,16 @@ type SMTP struct { Domain string `required:"true" default:"inbucket" desc:"HELO domain"` DomainNoStore string `desc:"Load testing domain"` MaxRecipients int `required:"true" default:"200" desc:"Maximum RCPT TO per message"` - MaxIdle time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` MaxMessageBytes int `required:"true" default:"2048000" desc:"Maximum message size"` StoreMessages bool `required:"true" default:"true" desc:"Store incoming mail?"` + Timeout time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` } // POP3 contains the POP3 server configuration. type POP3 struct { Addr string `required:"true" default:"0.0.0.0:1100" desc:"POP3 server IP4 host:port"` Domain string `required:"true" default:"inbucket" desc:"HELLO domain"` - MaxIdle time.Duration `required:"true" default:"600s" desc:"Idle network timeout"` + Timeout time.Duration `required:"true" default:"600s" desc:"Idle network timeout"` } // Web contains the HTTP server configuration. diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 00334fa..72d4355 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -496,7 +496,7 @@ func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { // Load the users mailbox func (ses *Session) loadMailbox() { - m, err := ses.server.dataStore.GetMessages(ses.user) + m, err := ses.server.store.GetMessages(ses.user) if err != nil { ses.logError("Failed to load messages for %v: %v", ses.user, err) } @@ -522,7 +522,7 @@ func (ses *Session) processDeletes() { for i, msg := range ses.messages { if !ses.retain[i] { ses.logTrace("Deleting %v", msg) - if err := ses.server.dataStore.RemoveMessage(ses.user, msg.ID()); err != nil { + if err := ses.server.store.RemoveMessage(ses.user, msg.ID()); err != nil { ses.logWarn("Error deleting %v: %v", msg, err) } } @@ -536,7 +536,7 @@ func (ses *Session) enterState(state State) { // Calculate the next read or write deadline based on maxIdleSeconds func (ses *Session) nextDeadline() time.Time { - return time.Now().Add(ses.server.maxIdle) + return time.Now().Add(ses.server.timeout) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index ee6d2da..2db2d00 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -15,20 +15,20 @@ import ( type Server struct { host string domain string - maxIdle time.Duration - dataStore storage.Store + timeout time.Duration + store storage.Store listener net.Listener globalShutdown chan bool waitgroup *sync.WaitGroup } // New creates a new Server struct -func New(cfg config.POP3, shutdownChan chan bool, ds storage.Store) *Server { +func New(cfg config.POP3, shutdownChan chan bool, store storage.Store) *Server { return &Server{ host: cfg.Addr, domain: cfg.Domain, - dataStore: ds, - maxIdle: cfg.MaxIdle, + store: store, + timeout: cfg.Timeout, globalShutdown: shutdownChan, waitgroup: new(sync.WaitGroup), } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 12338b2..634c56b 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -414,7 +414,7 @@ func (ss *Session) greet() { // Calculate the next read or write deadline based on maxIdle func (ss *Session) nextDeadline() time.Time { - return time.Now().Add(ss.server.maxIdle) + return time.Now().Add(ss.server.timeout) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index e047230..27fa663 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -366,7 +366,7 @@ func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown f Domain: "inbucket.local", DomainNoStore: "bitbucket.local", MaxRecipients: 5, - MaxIdle: 5, + Timeout: 5, MaxMessageBytes: 5000, StoreMessages: true, } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index e56ac93..339e32e 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -42,9 +42,9 @@ type Server struct { domain string domainNoStore string maxRecips int - maxIdle time.Duration maxMessageBytes int storeMessages bool + timeout time.Duration // Dependencies apolicy *policy.Addressing // Address policy. @@ -89,7 +89,7 @@ func NewServer( domain: cfg.Domain, domainNoStore: strings.ToLower(cfg.DomainNoStore), maxRecips: cfg.MaxRecipients, - maxIdle: cfg.MaxIdle, + timeout: cfg.Timeout, maxMessageBytes: cfg.MaxMessageBytes, storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, From 3c7c24b6980170f96110329f6c1b2dd8d661401e Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Thu, 22 Mar 2018 20:30:23 -0700 Subject: [PATCH 35/82] storage: Calculate size of store for status page --- CHANGELOG.md | 6 +++++- pkg/storage/retention.go | 9 +++++++++ themes/bootstrap/public/metrics.js | 2 ++ themes/bootstrap/templates/root/status.html | 6 ++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e674d8d..5fb1ecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,18 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added +- Store size is now calculated during retention scan and displayed on the Status + page. + ### Changed - Massive refactor of back-end code. Inbucket should now be both easier and more enjoyable to work on. + ## [v1.3.1] - 2018-03-10 ### Fixed - - Adding additional locking during message delivery to prevent race condition that could lose messages. diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index d083152..c17e725 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -18,14 +18,17 @@ var ( expRetentionDeletesTotal = new(expvar.Int) expRetentionPeriod = new(expvar.Int) expRetainedCurrent = new(expvar.Int) + expRetainedSize = new(expvar.Int) // History of certain stats retentionDeletesHist = list.New() retainedHist = list.New() + sizeHist = list.New() // History rendered as comma delimited string expRetentionDeletesHist = new(expvar.String) expRetainedHist = new(expvar.String) + expSizeHist = new(expvar.String) ) func init() { @@ -36,10 +39,13 @@ func init() { rm.Set("Period", expRetentionPeriod) rm.Set("RetainedHist", expRetainedHist) rm.Set("RetainedCurrent", expRetainedCurrent) + rm.Set("RetainedSize", expRetainedSize) + rm.Set("SizeHist", expSizeHist) log.AddTickerFunc(func() { expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) + expSizeHist.Set(log.PushMetric(sizeHist, expRetainedSize)) }) } @@ -118,6 +124,7 @@ func (rs *RetentionScanner) DoScan() error { log.Tracef("Starting retention scan") cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 + storeSize := int64(0) // Loop over all mailboxes. err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { @@ -130,6 +137,7 @@ func (rs *RetentionScanner) DoScan() error { } } else { retained++ + storeSize += msg.Size() } } select { @@ -147,6 +155,7 @@ func (rs *RetentionScanner) DoScan() error { // Update metrics setRetentionScanCompleted(time.Now()) expRetainedCurrent.Set(int64(retained)) + expRetainedSize.Set(storeSize) return nil } diff --git a/themes/bootstrap/public/metrics.js b/themes/bootstrap/public/metrics.js index f3fc0f1..ec8a5eb 100644 --- a/themes/bootstrap/public/metrics.js +++ b/themes/bootstrap/public/metrics.js @@ -120,6 +120,8 @@ function displayMetrics(data, textStatus, jqXHR) { setHistoryOfActivity('retentionDeletesTotal', data.retention.DeletesHist); metric('retainedCurrent', data.retention.RetainedCurrent, numberFilter, false); setHistoryOfCount('retainedCurrent', data.retention.RetainedHist); + metric('retainedSize', data.retention.RetainedSize, sizeFilter, false); + setHistoryOfCount('retainedSize', data.retention.SizeHist); } function loadMetrics() { diff --git a/themes/bootstrap/templates/root/status.html b/themes/bootstrap/templates/root/status.html index 31fbc35..3c708b1 100644 --- a/themes/bootstrap/templates/root/status.html +++ b/themes/bootstrap/templates/root/status.html @@ -203,6 +203,12 @@ $(document).ready(
+
+
Store Size:
+
.
+
+ +
From bb0fb410c1cb7822dcaa12b17967332cbea636ca Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Thu, 22 Mar 2018 22:02:08 -0700 Subject: [PATCH 36/82] mem: Initial in-memory store implementation for #88 - Reduce default retention sleep, change description. --- pkg/config/config.go | 2 +- pkg/storage/mem/message.go | 51 ++++++++++++ pkg/storage/mem/store.go | 148 ++++++++++++++++++++++++++++++++++ pkg/storage/mem/store_test.go | 17 ++++ 4 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 pkg/storage/mem/message.go create mode 100644 pkg/storage/mem/store.go create mode 100644 pkg/storage/mem/store_test.go diff --git a/pkg/config/config.go b/pkg/config/config.go index d6b4b68..25f805a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -71,7 +71,7 @@ type Web struct { type Storage struct { Path string `required:"true" default:"/tmp/inbucket" desc:"Mail store path"` RetentionPeriod time.Duration `required:"true" default:"24h" desc:"Duration to retain messages"` - RetentionSleep time.Duration `required:"true" default:"100ms" desc:"Duration to sleep between deletes"` + RetentionSleep time.Duration `required:"true" default:"50ms" desc:"Duration to sleep between mailboxes"` MailboxMsgCap int `required:"true" default:"500" desc:"Maximum messages per mailbox"` } diff --git a/pkg/storage/mem/message.go b/pkg/storage/mem/message.go new file mode 100644 index 0000000..ea0d351 --- /dev/null +++ b/pkg/storage/mem/message.go @@ -0,0 +1,51 @@ +package mem + +import ( + "bytes" + "io" + "io/ioutil" + "net/mail" + "time" + + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// Message is a memory store message. +type Message struct { + index int + mailbox string + id string + from *mail.Address + to []*mail.Address + date time.Time + subject string + source []byte +} + +var _ storage.Message = &Message{} + +// Mailbox returns the mailbox name. +func (m *Message) Mailbox() string { return m.mailbox } + +// ID the message ID. +func (m *Message) ID() string { return m.id } + +// From returns the from address. +func (m *Message) From() *mail.Address { return m.from } + +// To returns the to address list. +func (m *Message) To() []*mail.Address { return m.to } + +// Date returns the date received. +func (m *Message) Date() time.Time { return m.date } + +// Subject returns the subject line. +func (m *Message) Subject() string { return m.subject } + +// Source returns a reader for the message source. +func (m *Message) Source() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(m.source)), nil +} + +// Size returns the message size in bytes. +func (m *Message) Size() int64 { return int64(len(m.source)) } diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go new file mode 100644 index 0000000..f70d5be --- /dev/null +++ b/pkg/storage/mem/store.go @@ -0,0 +1,148 @@ +package mem + +import ( + "io/ioutil" + "sort" + "strconv" + "sync" + + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// Store implements an in-memory message store. +type Store struct { + sync.Mutex + boxes map[string]*mbox +} + +type mbox struct { + sync.RWMutex + name string + counter int + messages map[string]*Message +} + +var _ storage.Store = &Store{} + +// New returns an emtpy memory store. +func New() *Store { + return &Store{ + boxes: make(map[string]*mbox), + } +} + +// AddMessage stores the message, message ID and Size will be ignored. +func (s *Store) AddMessage(message storage.Message) (id string, err error) { + s.withMailbox(message.Mailbox(), true, func(mb *mbox) { + r, ierr := message.Source() + if ierr != nil { + err = ierr + return + } + source, ierr := ioutil.ReadAll(r) + if ierr != nil { + err = ierr + return + } + // Generate message ID. + mb.counter++ + id = strconv.Itoa(mb.counter) + m := &Message{ + index: mb.counter, + mailbox: message.Mailbox(), + id: id, + from: message.From(), + to: message.To(), + date: message.Date(), + subject: message.Subject(), + source: source, + } + mb.messages[id] = m + }) + return id, err +} + +// GetMessage gets a mesage. +func (s *Store) GetMessage(mailbox, id string) (m storage.Message, err error) { + s.withMailbox(mailbox, false, func(mb *mbox) { + m = mb.messages[id] + }) + return m, err +} + +// GetMessages gets a list of messages. +func (s *Store) GetMessages(mailbox string) (ms []storage.Message, err error) { + s.withMailbox(mailbox, false, func(mb *mbox) { + ms = make([]storage.Message, 0, len(mb.messages)) + for _, v := range mb.messages { + ms = append(ms, v) + } + sort.Slice(ms, func(i, j int) bool { + return ms[i].(*Message).index < ms[j].(*Message).index + }) + }) + return ms, err +} + +// PurgeMessages deletes the contents of a mailbox. +func (s *Store) PurgeMessages(mailbox string) error { + s.withMailbox(mailbox, true, func(mb *mbox) { + mb.messages = make(map[string]*Message) + }) + return nil +} + +// RemoveMessage deletes a single message. +func (s *Store) RemoveMessage(mailbox, id string) error { + s.withMailbox(mailbox, true, func(mb *mbox) { + delete(mb.messages, id) + }) + return nil +} + +// VisitMailboxes visits each mailbox in the store. +func (s *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { + // Lock store, get names of all mailboxes. + s.Lock() + boxNames := make([]string, 0, len(s.boxes)) + for k := range s.boxes { + boxNames = append(boxNames, k) + } + s.Unlock() + // Process mailboxes. + for _, mailbox := range boxNames { + ms, _ := s.GetMessages(mailbox) + if !f(ms) { + break + } + } + return nil +} + +// withMailbox gets or creates a mailbox, locks it, then calls f. +func (s *Store) withMailbox(mailbox string, rw bool, f func(mb *mbox)) { + s.Lock() + mb, ok := s.boxes[mailbox] + if !ok { + // Create mailbox + mb = &mbox{ + name: mailbox, + messages: make(map[string]*Message), + } + s.boxes[mailbox] = mb + } + s.Unlock() + if rw { + mb.Lock() + } else { + mb.RLock() + } + defer func() { + if rw { + mb.Unlock() + } else { + mb.RUnlock() + } + }() + f(mb) +} diff --git a/pkg/storage/mem/store_test.go b/pkg/storage/mem/store_test.go new file mode 100644 index 0000000..cbfdb05 --- /dev/null +++ b/pkg/storage/mem/store_test.go @@ -0,0 +1,17 @@ +package mem + +import ( + "testing" + + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" +) + +// TestSuite runs storage package test suite on file store. +func TestSuite(t *testing.T) { + test.StoreSuite(t, func() (storage.Store, func(), error) { + s := New() + destroy := func() {} + return s, destroy, nil + }) +} From 281cc21412b037237f5f6a0e251c9ff9eeb5cb45 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 24 Mar 2018 13:18:51 -0700 Subject: [PATCH 37/82] storage: Make type/params configurable for #88 --- cmd/inbucket/main.go | 12 +++++++++++- pkg/config/config.go | 9 +++++---- pkg/storage/file/fstore.go | 9 ++++----- pkg/storage/file/fstore_test.go | 15 ++++++++++----- pkg/storage/mem/store.go | 5 +++-- pkg/storage/mem/store_test.go | 3 ++- pkg/storage/storage.go | 14 ++++++++++++++ themes/bootstrap/templates/root/status.html | 4 ++++ 8 files changed, 53 insertions(+), 18 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index a54a8d8..17ccb0e 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -23,6 +23,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/storage/file" + "github.com/jhillyerd/inbucket/pkg/storage/mem" "github.com/jhillyerd/inbucket/pkg/webui" ) @@ -45,6 +46,10 @@ func init() { expvar.Publish("goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() })) + + // Register storage implementations. + storage.Constructors["file"] = file.New + storage.Constructors["memory"] = mem.New } func main() { @@ -97,8 +102,13 @@ func main() { // Configure internal services. rootCtx, rootCancel := context.WithCancel(context.Background()) shutdownChan := make(chan bool) + store, err := storage.FromConfig(conf.Storage) + if err != nil { + log.Errorf("Fatal storage error: %v", err) + removePIDFile(*pidfile) + os.Exit(1) + } msgHub := msghub.New(rootCtx, conf.Web.MonitorHistory) - store := file.New(conf.Storage) addrPolicy := &policy.Addressing{Config: conf.SMTP} mmanager := &message.StoreManager{Store: store, Hub: msgHub} // Start Retention scanner. diff --git a/pkg/config/config.go b/pkg/config/config.go index 25f805a..24017f4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -69,10 +69,11 @@ type Web struct { // Storage contains the mail store configuration. type Storage struct { - Path string `required:"true" default:"/tmp/inbucket" desc:"Mail store path"` - RetentionPeriod time.Duration `required:"true" default:"24h" desc:"Duration to retain messages"` - RetentionSleep time.Duration `required:"true" default:"50ms" desc:"Duration to sleep between mailboxes"` - MailboxMsgCap int `required:"true" default:"500" desc:"Maximum messages per mailbox"` + Type string `required:"true" default:"memory" desc:"Storage impl: file or memory"` + Params map[string]string `desc:"Storage impl parameters, see docs."` + RetentionPeriod time.Duration `required:"true" default:"24h" desc:"Duration to retain messages"` + RetentionSleep time.Duration `required:"true" default:"50ms" desc:"Duration to sleep between mailboxes"` + MailboxMsgCap int `required:"true" default:"500" desc:"Maximum messages per mailbox"` } // Process loads and parses configuration from the environment. diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 069a3fe..fa00f3c 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -48,11 +48,10 @@ type Store struct { } // New creates a new DataStore object using the specified path -func New(cfg config.Storage) storage.Store { - path := cfg.Path +func New(cfg config.Storage) (storage.Store, error) { + path := cfg.Params["path"] if path == "" { - log.Errorf("No value configured for datastore path") - return nil + return nil, fmt.Errorf("'path' parameter not specified") } mailPath := filepath.Join(path, "mail") if _, err := os.Stat(mailPath); err != nil { @@ -61,7 +60,7 @@ func New(cfg config.Storage) storage.Store { log.Errorf("Error creating dir %q: %v", mailPath, err) } } - return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} + return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap}, nil } // AddMessage adds a message to the specified mailbox. diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 0b42e86..10326b0 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -265,13 +265,18 @@ func setupDataStore(cfg config.Storage) (*Store, *bytes.Buffer) { if err != nil { panic(err) } - - // Capture log output + // Capture log output. buf := new(bytes.Buffer) log.SetOutput(buf) - - cfg.Path = path - return New(cfg).(*Store), buf + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + cfg.Params["path"] = path + s, err := New(cfg) + if err != nil { + panic(err) + } + return s.(*Store), buf } // deliverMessage creates and delivers a message to the specific mailbox, returning diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go index f70d5be..cbf78b5 100644 --- a/pkg/storage/mem/store.go +++ b/pkg/storage/mem/store.go @@ -6,6 +6,7 @@ import ( "strconv" "sync" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -25,10 +26,10 @@ type mbox struct { var _ storage.Store = &Store{} // New returns an emtpy memory store. -func New() *Store { +func New(cfg config.Storage) (storage.Store, error) { return &Store{ boxes: make(map[string]*mbox), - } + }, nil } // AddMessage stores the message, message ID and Size will be ignored. diff --git a/pkg/storage/mem/store_test.go b/pkg/storage/mem/store_test.go index cbfdb05..53f986d 100644 --- a/pkg/storage/mem/store_test.go +++ b/pkg/storage/mem/store_test.go @@ -3,6 +3,7 @@ package mem import ( "testing" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -10,7 +11,7 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { test.StoreSuite(t, func() (storage.Store, func(), error) { - s := New() + s, _ := New(config.Storage{}) destroy := func() {} return s, destroy, nil }) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index edc4afd..67ef9fc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -3,9 +3,12 @@ package storage import ( "errors" + "fmt" "io" "net/mail" "time" + + "github.com/jhillyerd/inbucket/pkg/config" ) var ( @@ -14,6 +17,9 @@ var ( // ErrNotWritable indicates the message is closed; no longer writable ErrNotWritable = errors.New("Message not writable") + + // Constructors tracks registered storage constructors + Constructors = make(map[string]func(config.Storage) (Store, error)) ) // Store is the interface Inbucket uses to interact with storage implementations. @@ -38,3 +44,11 @@ type Message interface { Source() (io.ReadCloser, error) Size() int64 } + +// FromConfig creates an instance of the Store based on the provided configuration. +func FromConfig(c config.Storage) (store Store, err error) { + if cf := Constructors[c.Type]; cf != nil { + return cf(c) + } + return nil, fmt.Errorf("unknown storage type configured: %q", c.Type) +} diff --git a/themes/bootstrap/templates/root/status.html b/themes/bootstrap/templates/root/status.html index 3c708b1..f666ee7 100644 --- a/themes/bootstrap/templates/root/status.html +++ b/themes/bootstrap/templates/root/status.html @@ -171,6 +171,10 @@ $(document).ready(
+
+
Store Type:
+
{{ .storageConfig.Type }}
+
Retention Period:
From b42ea130eac7b7928ca1a0d249f67c3e2acaf6c8 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 24 Mar 2018 14:30:53 -0700 Subject: [PATCH 38/82] storage/mem: implement message cap for #88 - Move message cap tests into storage test suite. - Update change log. --- CHANGELOG.md | 5 +++ pkg/storage/file/fstore_test.go | 76 +-------------------------------- pkg/storage/mem/store.go | 24 ++++++++--- pkg/storage/mem/store_test.go | 4 +- pkg/test/storage_suite.go | 59 +++++++++++++++++++++---- 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fb1ecc..2c226d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added +- Inbucket is now configured using environment variables instead of a config + file. +- In-memory storage option, best for small installations and desktops. Will be + used by default. +- Storage type is now displayed on Status page. - Store size is now calculated during retention scan and displayed on the Status page. diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 10326b0..7023706 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -22,8 +22,8 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { - test.StoreSuite(t, func() (storage.Store, func(), error) { - ds, _ := setupDataStore(config.Storage{}) + test.StoreSuite(t, func(conf config.Storage) (storage.Store, func(), error) { + ds, _ := setupDataStore(conf) destroy := func() { teardownDataStore(ds) } @@ -144,78 +144,6 @@ func TestFSMissing(t *testing.T) { } } -// Test delivering several messages to the same mailbox, see if message cap works -func TestFSMessageCap(t *testing.T) { - mbCap := 10 - ds, logbuf := setupDataStore(config.Storage{MailboxMsgCap: mbCap}) - defer teardownDataStore(ds) - - mbName := "captain" - for i := 0; i < 20; i++ { - // Add a message - subj := fmt.Sprintf("subject %v", i) - deliverMessage(ds, mbName, subj, time.Now()) - t.Logf("Delivered %q", subj) - - // Check number of messages - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - if len(msgs) > mbCap { - t.Errorf("Mailbox should be capped at %v messages, but has %v", mbCap, len(msgs)) - } - - // Check that the first message is correct - first := i - mbCap + 1 - if first < 0 { - first = 0 - } - firstSubj := fmt.Sprintf("subject %v", first) - if firstSubj != msgs[0].Subject() { - t.Errorf("Expected first subject to be %q, got %q", firstSubj, msgs[0].Subject()) - } - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test delivering several messages to the same mailbox, see if no message cap works -func TestFSNoMessageCap(t *testing.T) { - mbCap := 0 - ds, logbuf := setupDataStore(config.Storage{MailboxMsgCap: mbCap}) - defer teardownDataStore(ds) - - mbName := "captain" - for i := 0; i < 20; i++ { - // Add a message - subj := fmt.Sprintf("subject %v", i) - deliverMessage(ds, mbName, subj, time.Now()) - t.Logf("Delivered %q", subj) - - // Check number of messages - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - if len(msgs) != i+1 { - t.Errorf("Expected %v messages, got %v", i+1, len(msgs)) - } - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - // Test Get the latest message func TestGetLatestMessage(t *testing.T) { ds, logbuf := setupDataStore(config.Storage{}) diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go index cbf78b5..0559a5b 100644 --- a/pkg/storage/mem/store.go +++ b/pkg/storage/mem/store.go @@ -14,12 +14,14 @@ import ( type Store struct { sync.Mutex boxes map[string]*mbox + cap int } type mbox struct { sync.RWMutex name string - counter int + last int + first int messages map[string]*Message } @@ -29,6 +31,7 @@ var _ storage.Store = &Store{} func New(cfg config.Storage) (storage.Store, error) { return &Store{ boxes: make(map[string]*mbox), + cap: cfg.MailboxMsgCap, }, nil } @@ -46,10 +49,10 @@ func (s *Store) AddMessage(message storage.Message) (id string, err error) { return } // Generate message ID. - mb.counter++ - id = strconv.Itoa(mb.counter) + mb.last++ + id = strconv.Itoa(mb.last) m := &Message{ - index: mb.counter, + index: mb.last, mailbox: message.Mailbox(), id: id, from: message.From(), @@ -59,6 +62,13 @@ func (s *Store) AddMessage(message storage.Message) (id string, err error) { source: source, } mb.messages[id] = m + if s.cap > 0 { + // Enforce cap. + for len(mb.messages) > s.cap { + delete(mb.messages, strconv.Itoa(mb.first)) + mb.first++ + } + } }) return id, err } @@ -121,7 +131,7 @@ func (s *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { } // withMailbox gets or creates a mailbox, locks it, then calls f. -func (s *Store) withMailbox(mailbox string, rw bool, f func(mb *mbox)) { +func (s *Store) withMailbox(mailbox string, writeLock bool, f func(mb *mbox)) { s.Lock() mb, ok := s.boxes[mailbox] if !ok { @@ -133,13 +143,13 @@ func (s *Store) withMailbox(mailbox string, rw bool, f func(mb *mbox)) { s.boxes[mailbox] = mb } s.Unlock() - if rw { + if writeLock { mb.Lock() } else { mb.RLock() } defer func() { - if rw { + if writeLock { mb.Unlock() } else { mb.RUnlock() diff --git a/pkg/storage/mem/store_test.go b/pkg/storage/mem/store_test.go index 53f986d..8a45ec1 100644 --- a/pkg/storage/mem/store_test.go +++ b/pkg/storage/mem/store_test.go @@ -10,8 +10,8 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { - test.StoreSuite(t, func() (storage.Store, func(), error) { - s, _ := New(config.Storage{}) + test.StoreSuite(t, func(conf config.Storage) (storage.Store, func(), error) { + s, _ := New(conf) destroy := func() {} return s, destroy, nil }) diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 3d167b5..1038dac 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -9,30 +9,34 @@ import ( "testing" "time" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" ) // StoreFactory returns a new store for the test suite. -type StoreFactory func() (store storage.Store, destroy func(), err error) +type StoreFactory func(config.Storage) (store storage.Store, destroy func(), err error) // StoreSuite runs a set of general tests on the provided Store. func StoreSuite(t *testing.T, factory StoreFactory) { testCases := []struct { name string test func(*testing.T, storage.Store) + conf config.Storage }{ - {"metadata", testMetadata}, - {"content", testContent}, - {"delivery order", testDeliveryOrder}, - {"size", testSize}, - {"delete", testDelete}, - {"purge", testPurge}, - {"visit mailboxes", testVisitMailboxes}, + {"metadata", testMetadata, config.Storage{}}, + {"content", testContent, config.Storage{}}, + {"delivery order", testDeliveryOrder, config.Storage{}}, + {"size", testSize, config.Storage{}}, + {"delete", testDelete, config.Storage{}}, + {"purge", testPurge, config.Storage{}}, + {"cap=10", testMsgCap, config.Storage{MailboxMsgCap: 10}}, + {"cap=0", testNoMsgCap, config.Storage{MailboxMsgCap: 0}}, + {"visit mailboxes", testVisitMailboxes, config.Storage{}}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - store, destroy, err := factory() + store, destroy, err := factory(tc.conf) if err != nil { t.Fatal(err) } @@ -260,6 +264,43 @@ func testPurge(t *testing.T, store storage.Store) { getAndCountMessages(t, store, mailbox, 0) } +// testMsgCap verifies the message cap is enforced. +func testMsgCap(t *testing.T, store storage.Store) { + mbCap := 10 + mailbox := "captain" + for i := 0; i < 20; i++ { + subj := fmt.Sprintf("subject %v", i) + deliverMessage(t, store, mailbox, subj, time.Now()) + msgs, err := store.GetMessages(mailbox) + if err != nil { + t.Fatalf("Failed to GetMessages for %q: %v", mailbox, err) + } + if len(msgs) > mbCap { + t.Errorf("Mailbox has %v messages, should be capped at %v", len(msgs), mbCap) + break + } + // Check that the first message is correct. + first := i - mbCap + 1 + if first < 0 { + first = 0 + } + firstSubj := fmt.Sprintf("subject %v", first) + if firstSubj != msgs[0].Subject() { + t.Errorf("Got subject %q, wanted first subject: %q", msgs[0].Subject(), firstSubj) + } + } +} + +// testNoMsgCap verfies a cap of 0 is not enforced. +func testNoMsgCap(t *testing.T, store storage.Store) { + mailbox := "captain" + for i := 0; i < 20; i++ { + subj := fmt.Sprintf("subject %v", i) + deliverMessage(t, store, mailbox, subj, time.Now()) + getAndCountMessages(t, store, mailbox, i+1) + } +} + // testVisitMailboxes creates some mailboxes and confirms the VisitMailboxes method visits all of // them. func testVisitMailboxes(t *testing.T, ds storage.Store) { From 412b62d6fad94c33329285f5449eabd9769cb0d2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 24 Mar 2018 20:27:05 -0700 Subject: [PATCH 39/82] storage/mem: implement size enforcer for #88 --- pkg/storage/mem/maxsize.go | 73 +++++++++++++++++++++++++++ pkg/storage/mem/message.go | 2 + pkg/storage/mem/store.go | 92 +++++++++++++++++++++++++---------- pkg/storage/mem/store_test.go | 64 ++++++++++++++++++++++++ pkg/test/storage_suite.go | 42 ++++++++-------- 5 files changed, 225 insertions(+), 48 deletions(-) create mode 100644 pkg/storage/mem/maxsize.go diff --git a/pkg/storage/mem/maxsize.go b/pkg/storage/mem/maxsize.go new file mode 100644 index 0000000..888247b --- /dev/null +++ b/pkg/storage/mem/maxsize.go @@ -0,0 +1,73 @@ +package mem + +import "container/list" + +type msgDone struct { + msg *Message + done chan struct{} +} + +// enforceMaxSize will delete the oldest message until the entire mail store is equal to or less +// than Store.maxSize bytes. +func (s *Store) maxSizeEnforcer(maxSize int64) { + all := &list.List{} + curSize := int64(0) + for { + select { + case md, ok := <-s.incoming: + if !ok { + return + } + // Add message to all. + m := md.msg + el := all.PushBack(m) + m.el = el + curSize += int64(m.Size()) + for curSize > maxSize { + // Remove oldest message. + el := all.Front() + all.Remove(el) + m := el.Value.(*Message) + if s.removeMessage(m.mailbox, m.id) != nil { + curSize -= int64(m.Size()) + } + } + close(md.done) + case md, ok := <-s.remove: + if !ok { + return + } + // Remove message from all. + m := md.msg + el := all.Remove(m.el) + if el != nil { + curSize -= int64(m.Size()) + } + close(md.done) + } + } +} + +// enforcerDeliver sends delivery to enforcer if configured, and waits for completion. +func (s *Store) enforcerDeliver(m *Message) { + if s.incoming != nil { + md := &msgDone{ + msg: m, + done: make(chan struct{}), + } + s.incoming <- md + <-md.done + } +} + +// enforcerRemove sends removal to enforcer if configured, and waits for completion. +func (s *Store) enforcerRemove(m *Message) { + if s.remove != nil { + md := &msgDone{ + msg: m, + done: make(chan struct{}), + } + s.remove <- md + <-md.done + } +} diff --git a/pkg/storage/mem/message.go b/pkg/storage/mem/message.go index ea0d351..b5ca498 100644 --- a/pkg/storage/mem/message.go +++ b/pkg/storage/mem/message.go @@ -2,6 +2,7 @@ package mem import ( "bytes" + "container/list" "io" "io/ioutil" "net/mail" @@ -20,6 +21,7 @@ type Message struct { date time.Time subject string source []byte + el *list.Element // This message in Store.messages } var _ storage.Message = &Message{} diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go index 0559a5b..c37b241 100644 --- a/pkg/storage/mem/store.go +++ b/pkg/storage/mem/store.go @@ -1,6 +1,7 @@ package mem import ( + "fmt" "io/ioutil" "sort" "strconv" @@ -13,8 +14,10 @@ import ( // Store implements an in-memory message store. type Store struct { sync.Mutex - boxes map[string]*mbox - cap int + boxes map[string]*mbox + cap int // Per-mailbox message cap. + incoming chan *msgDone // New messages for size enforcer. + remove chan *msgDone // Remove deleted messages from size enforcer. } type mbox struct { @@ -29,38 +32,51 @@ var _ storage.Store = &Store{} // New returns an emtpy memory store. func New(cfg config.Storage) (storage.Store, error) { - return &Store{ + s := &Store{ boxes: make(map[string]*mbox), cap: cfg.MailboxMsgCap, - }, nil + } + if str, ok := cfg.Params["maxkb"]; ok { + maxKB, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse maxkb: %v", err) + } + if maxKB > 0 { + // Setup enforcer. + s.incoming = make(chan *msgDone) + s.remove = make(chan *msgDone) + go s.maxSizeEnforcer(maxKB * 1024) + } + } + return s, nil } // AddMessage stores the message, message ID and Size will be ignored. func (s *Store) AddMessage(message storage.Message) (id string, err error) { + r, ierr := message.Source() + if ierr != nil { + err = ierr + return + } + source, ierr := ioutil.ReadAll(r) + if ierr != nil { + err = ierr + return + } + m := &Message{ + mailbox: message.Mailbox(), + from: message.From(), + to: message.To(), + date: message.Date(), + subject: message.Subject(), + } s.withMailbox(message.Mailbox(), true, func(mb *mbox) { - r, ierr := message.Source() - if ierr != nil { - err = ierr - return - } - source, ierr := ioutil.ReadAll(r) - if ierr != nil { - err = ierr - return - } // Generate message ID. mb.last++ + m.index = mb.last id = strconv.Itoa(mb.last) - m := &Message{ - index: mb.last, - mailbox: message.Mailbox(), - id: id, - from: message.From(), - to: message.To(), - date: message.Date(), - subject: message.Subject(), - source: source, - } + m.id = id + m.source = source mb.messages[id] = m if s.cap > 0 { // Enforce cap. @@ -70,6 +86,7 @@ func (s *Store) AddMessage(message storage.Message) (id string, err error) { } } }) + s.enforcerDeliver(m) return id, err } @@ -97,17 +114,38 @@ func (s *Store) GetMessages(mailbox string) (ms []storage.Message, err error) { // PurgeMessages deletes the contents of a mailbox. func (s *Store) PurgeMessages(mailbox string) error { + var messages map[string]*Message s.withMailbox(mailbox, true, func(mb *mbox) { + messages = mb.messages mb.messages = make(map[string]*Message) }) + if len(messages) > 0 && s.remove != nil { + for _, m := range messages { + s.enforcerRemove(m) + } + } return nil } +// removeMessage deletes a single message without notifying the size enforcer. Returns the message +// that was removed. +func (s *Store) removeMessage(mailbox, id string) *Message { + var m *Message + s.withMailbox(mailbox, true, func(mb *mbox) { + m = mb.messages[id] + if m != nil { + delete(mb.messages, id) + } + }) + return m +} + // RemoveMessage deletes a single message. func (s *Store) RemoveMessage(mailbox, id string) error { - s.withMailbox(mailbox, true, func(mb *mbox) { - delete(mb.messages, id) - }) + m := s.removeMessage(mailbox, id) + if m != nil { + s.enforcerRemove(m) + } return nil } diff --git a/pkg/storage/mem/store_test.go b/pkg/storage/mem/store_test.go index 8a45ec1..c48bca8 100644 --- a/pkg/storage/mem/store_test.go +++ b/pkg/storage/mem/store_test.go @@ -1,7 +1,9 @@ package mem import ( + "sync" "testing" + "time" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/storage" @@ -16,3 +18,65 @@ func TestSuite(t *testing.T) { return s, destroy, nil }) } + +// TestMessageList verifies the operation of the global message list: mem.Store.messages. +func TestMaxSize(t *testing.T) { + maxSize := int64(2048) + s, _ := New(config.Storage{Params: map[string]string{"maxkb": "2"}}) + boxes := []string{"alpha", "beta", "whiskey", "tango", "foxtrot"} + n := 10 + // total := 50 + sizeChan := make(chan int64, len(boxes)) + // Populate mailboxes concurrently. + for _, mailbox := range boxes { + go func(mailbox string) { + size := int64(0) + for i := 0; i < n; i++ { + _, nbytes := test.DeliverToStore(t, s, mailbox, "subject", time.Now()) + size += nbytes + } + sizeChan <- size + }(mailbox) + } + // Wait for sizes. + sentBytesTotal := int64(0) + for range boxes { + sentBytesTotal += <-sizeChan + } + // Calculate actual size. + gotSize := int64(0) + s.VisitMailboxes(func(messages []storage.Message) bool { + for _, m := range messages { + gotSize += m.Size() + } + return true + }) + // Verify state. Messages are ~75 bytes each. + if gotSize < 2048-75 { + t.Errorf("Got total size %v, want greater than: %v", gotSize, 2048-75) + } + if gotSize > maxSize { + t.Errorf("Got total size %v, want less than: %v", gotSize, maxSize) + } + // Purge all messages concurrently, testing for deadlocks. + wg := &sync.WaitGroup{} + wg.Add(len(boxes)) + for _, mailbox := range boxes { + go func(mailbox string) { + err := s.PurgeMessages(mailbox) + if err != nil { + t.Fatal(err) + } + wg.Done() + }(mailbox) + } + wg.Wait() + count := 0 + s.VisitMailboxes(func(messages []storage.Message) bool { + count += len(messages) + return true + }) + if count != 0 { + t.Errorf("Got %v total messages, want: %v", count, 0) + } +} diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 1038dac..9936145 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -173,11 +173,11 @@ func testDeliveryOrder(t *testing.T, store storage.Store) { subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} for i, subj := range subjects { // Check mailbox count. - getAndCountMessages(t, store, mailbox, i) - deliverMessage(t, store, mailbox, subj, time.Now()) + GetAndCountMessages(t, store, mailbox, i) + DeliverToStore(t, store, mailbox, subj, time.Now()) } // Confirm delivery order. - msgs := getAndCountMessages(t, store, mailbox, 5) + msgs := GetAndCountMessages(t, store, mailbox, 5) for i, want := range subjects { got := msgs[i].Subject() if got != want { @@ -193,7 +193,7 @@ func testSize(t *testing.T, store storage.Store) { sentIds := make([]string, len(subjects)) sentSizes := make([]int64, len(subjects)) for i, subj := range subjects { - id, size := deliverMessage(t, store, mailbox, subj, time.Now()) + id, size := DeliverToStore(t, store, mailbox, subj, time.Now()) sentIds[i] = id sentSizes[i] = size } @@ -215,9 +215,9 @@ func testDelete(t *testing.T, store storage.Store) { mailbox := "fred" subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} for _, subj := range subjects { - deliverMessage(t, store, mailbox, subj, time.Now()) + DeliverToStore(t, store, mailbox, subj, time.Now()) } - msgs := getAndCountMessages(t, store, mailbox, len(subjects)) + msgs := GetAndCountMessages(t, store, mailbox, len(subjects)) // Delete a couple messages. err := store.RemoveMessage(mailbox, msgs[1].ID()) if err != nil { @@ -229,7 +229,7 @@ func testDelete(t *testing.T, store storage.Store) { } // Confirm deletion. subjects = []string{"alpha", "charlie", "echo"} - msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + msgs = GetAndCountMessages(t, store, mailbox, len(subjects)) for i, want := range subjects { got := msgs[i].Subject() if got != want { @@ -237,9 +237,9 @@ func testDelete(t *testing.T, store storage.Store) { } } // Try appending one more. - deliverMessage(t, store, mailbox, "foxtrot", time.Now()) + DeliverToStore(t, store, mailbox, "foxtrot", time.Now()) subjects = []string{"alpha", "charlie", "echo", "foxtrot"} - msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + msgs = GetAndCountMessages(t, store, mailbox, len(subjects)) for i, want := range subjects { got := msgs[i].Subject() if got != want { @@ -253,15 +253,15 @@ func testPurge(t *testing.T, store storage.Store) { mailbox := "fred" subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} for _, subj := range subjects { - deliverMessage(t, store, mailbox, subj, time.Now()) + DeliverToStore(t, store, mailbox, subj, time.Now()) } - getAndCountMessages(t, store, mailbox, len(subjects)) + GetAndCountMessages(t, store, mailbox, len(subjects)) // Purge and verify. err := store.PurgeMessages(mailbox) if err != nil { t.Fatal(err) } - getAndCountMessages(t, store, mailbox, 0) + GetAndCountMessages(t, store, mailbox, 0) } // testMsgCap verifies the message cap is enforced. @@ -270,7 +270,7 @@ func testMsgCap(t *testing.T, store storage.Store) { mailbox := "captain" for i := 0; i < 20; i++ { subj := fmt.Sprintf("subject %v", i) - deliverMessage(t, store, mailbox, subj, time.Now()) + DeliverToStore(t, store, mailbox, subj, time.Now()) msgs, err := store.GetMessages(mailbox) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mailbox, err) @@ -296,8 +296,8 @@ func testNoMsgCap(t *testing.T, store storage.Store) { mailbox := "captain" for i := 0; i < 20; i++ { subj := fmt.Sprintf("subject %v", i) - deliverMessage(t, store, mailbox, subj, time.Now()) - getAndCountMessages(t, store, mailbox, i+1) + DeliverToStore(t, store, mailbox, subj, time.Now()) + GetAndCountMessages(t, store, mailbox, i+1) } } @@ -306,8 +306,8 @@ func testNoMsgCap(t *testing.T, store storage.Store) { func testVisitMailboxes(t *testing.T, ds storage.Store) { boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} for _, name := range boxes { - deliverMessage(t, ds, name, "Old Message", time.Now().Add(-24*time.Hour)) - deliverMessage(t, ds, name, "New Message", time.Now()) + DeliverToStore(t, ds, name, "Old Message", time.Now().Add(-24*time.Hour)) + DeliverToStore(t, ds, name, "New Message", time.Now()) } seen := 0 err := ds.VisitMailboxes(func(messages []storage.Message) bool { @@ -326,9 +326,9 @@ func testVisitMailboxes(t *testing.T, ds storage.Store) { } } -// deliverMessage creates and delivers a message to the specific mailbox, returning the size of the +// DeliverToStore creates and delivers a message to the specific mailbox, returning the size of the // generated message. -func deliverMessage( +func DeliverToStore( t *testing.T, store storage.Store, mailbox string, @@ -356,9 +356,9 @@ func deliverMessage( return id, int64(len(testMsg)) } -// getAndCountMessages is a test helper that expects to receive count messages or fails the test, it +// GetAndCountMessages is a test helper that expects to receive count messages or fails the test, it // also checks return error. -func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.Message { +func GetAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.Message { t.Helper() msgs, err := s.GetMessages(mailbox) if err != nil { From b50c9267451a5a7d1957f5a7a27c9053a70c5b20 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 11:21:53 -0700 Subject: [PATCH 40/82] webui: Renamed themes dir to ui - Eliminated intermediate bootstrap dir --- CHANGELOG.md | 2 ++ pkg/config/config.go | 6 +++--- {themes => ui}/greeting.html | 0 {themes/bootstrap/public => ui/static}/bower.json | 0 .../static}/bower_components/bootstrap/.bower.json | 0 .../static}/bower_components/bootstrap/CHANGELOG.md | 0 .../static}/bower_components/bootstrap/Gemfile | 0 .../static}/bower_components/bootstrap/Gemfile.lock | 0 .../static}/bower_components/bootstrap/Gruntfile.js | 0 .../bower_components/bootstrap/ISSUE_TEMPLATE.md | 0 .../static}/bower_components/bootstrap/LICENSE | 0 .../static}/bower_components/bootstrap/README.md | 0 .../static}/bower_components/bootstrap/bower.json | 0 .../bootstrap/dist/css/bootstrap-theme.css | 0 .../bootstrap/dist/css/bootstrap-theme.css.map | 0 .../bootstrap/dist/css/bootstrap-theme.min.css | 0 .../bootstrap/dist/css/bootstrap-theme.min.css.map | 0 .../bootstrap/dist/css/bootstrap.css | 0 .../bootstrap/dist/css/bootstrap.css.map | 0 .../bootstrap/dist/css/bootstrap.min.css | 0 .../bootstrap/dist/css/bootstrap.min.css.map | 0 .../dist/fonts/glyphicons-halflings-regular.eot | Bin .../dist/fonts/glyphicons-halflings-regular.svg | 0 .../dist/fonts/glyphicons-halflings-regular.ttf | Bin .../dist/fonts/glyphicons-halflings-regular.woff | Bin .../dist/fonts/glyphicons-halflings-regular.woff2 | Bin .../bower_components/bootstrap/dist/js/bootstrap.js | 0 .../bootstrap/dist/js/bootstrap.min.js | 0 .../bower_components/bootstrap/dist/js/npm.js | 0 .../fonts/glyphicons-halflings-regular.eot | Bin .../fonts/glyphicons-halflings-regular.svg | 0 .../fonts/glyphicons-halflings-regular.ttf | Bin .../fonts/glyphicons-halflings-regular.woff | Bin .../fonts/glyphicons-halflings-regular.woff2 | Bin .../bower_components/bootstrap/grunt/.jshintrc | 0 .../bootstrap/grunt/bs-commonjs-generator.js | 0 .../bootstrap/grunt/bs-glyphicons-data-generator.js | 0 .../bootstrap/grunt/bs-lessdoc-parser.js | 0 .../bootstrap/grunt/bs-raw-files-generator.js | 0 .../bootstrap/grunt/change-version.js | 0 .../bootstrap/grunt/configBridge.json | 0 .../bootstrap/grunt/npm-shrinkwrap.json | 0 .../bootstrap/grunt/sauce_browsers.yml | 0 .../static}/bower_components/bootstrap/js/.jscsrc | 0 .../static}/bower_components/bootstrap/js/.jshintrc | 0 .../static}/bower_components/bootstrap/js/affix.js | 0 .../static}/bower_components/bootstrap/js/alert.js | 0 .../static}/bower_components/bootstrap/js/button.js | 0 .../bower_components/bootstrap/js/carousel.js | 0 .../bower_components/bootstrap/js/collapse.js | 0 .../bower_components/bootstrap/js/dropdown.js | 0 .../static}/bower_components/bootstrap/js/modal.js | 0 .../bower_components/bootstrap/js/popover.js | 0 .../bower_components/bootstrap/js/scrollspy.js | 0 .../static}/bower_components/bootstrap/js/tab.js | 0 .../bower_components/bootstrap/js/tooltip.js | 0 .../bower_components/bootstrap/js/transition.js | 0 .../bower_components/bootstrap/less/.csscomb.json | 0 .../bower_components/bootstrap/less/.csslintrc | 0 .../bower_components/bootstrap/less/alerts.less | 0 .../bower_components/bootstrap/less/badges.less | 0 .../bower_components/bootstrap/less/bootstrap.less | 0 .../bootstrap/less/breadcrumbs.less | 0 .../bootstrap/less/button-groups.less | 0 .../bower_components/bootstrap/less/buttons.less | 0 .../bower_components/bootstrap/less/carousel.less | 0 .../bower_components/bootstrap/less/close.less | 0 .../bower_components/bootstrap/less/code.less | 0 .../bootstrap/less/component-animations.less | 0 .../bower_components/bootstrap/less/dropdowns.less | 0 .../bower_components/bootstrap/less/forms.less | 0 .../bower_components/bootstrap/less/glyphicons.less | 0 .../bower_components/bootstrap/less/grid.less | 0 .../bootstrap/less/input-groups.less | 0 .../bower_components/bootstrap/less/jumbotron.less | 0 .../bower_components/bootstrap/less/labels.less | 0 .../bower_components/bootstrap/less/list-group.less | 0 .../bower_components/bootstrap/less/media.less | 0 .../bower_components/bootstrap/less/mixins.less | 0 .../bootstrap/less/mixins/alerts.less | 0 .../bootstrap/less/mixins/background-variant.less | 0 .../bootstrap/less/mixins/border-radius.less | 0 .../bootstrap/less/mixins/buttons.less | 0 .../bootstrap/less/mixins/center-block.less | 0 .../bootstrap/less/mixins/clearfix.less | 0 .../bootstrap/less/mixins/forms.less | 0 .../bootstrap/less/mixins/gradients.less | 0 .../bootstrap/less/mixins/grid-framework.less | 0 .../bootstrap/less/mixins/grid.less | 0 .../bootstrap/less/mixins/hide-text.less | 0 .../bootstrap/less/mixins/image.less | 0 .../bootstrap/less/mixins/labels.less | 0 .../bootstrap/less/mixins/list-group.less | 0 .../bootstrap/less/mixins/nav-divider.less | 0 .../bootstrap/less/mixins/nav-vertical-align.less | 0 .../bootstrap/less/mixins/opacity.less | 0 .../bootstrap/less/mixins/pagination.less | 0 .../bootstrap/less/mixins/panels.less | 0 .../bootstrap/less/mixins/progress-bar.less | 0 .../bootstrap/less/mixins/reset-filter.less | 0 .../bootstrap/less/mixins/reset-text.less | 0 .../bootstrap/less/mixins/resize.less | 0 .../less/mixins/responsive-visibility.less | 0 .../bootstrap/less/mixins/size.less | 0 .../bootstrap/less/mixins/tab-focus.less | 0 .../bootstrap/less/mixins/table-row.less | 0 .../bootstrap/less/mixins/text-emphasis.less | 0 .../bootstrap/less/mixins/text-overflow.less | 0 .../bootstrap/less/mixins/vendor-prefixes.less | 0 .../bower_components/bootstrap/less/modals.less | 0 .../bower_components/bootstrap/less/navbar.less | 0 .../bower_components/bootstrap/less/navs.less | 0 .../bower_components/bootstrap/less/normalize.less | 0 .../bower_components/bootstrap/less/pager.less | 0 .../bower_components/bootstrap/less/pagination.less | 0 .../bower_components/bootstrap/less/panels.less | 0 .../bower_components/bootstrap/less/popovers.less | 0 .../bower_components/bootstrap/less/print.less | 0 .../bootstrap/less/progress-bars.less | 0 .../bootstrap/less/responsive-embed.less | 0 .../bootstrap/less/responsive-utilities.less | 0 .../bootstrap/less/scaffolding.less | 0 .../bower_components/bootstrap/less/tables.less | 0 .../bower_components/bootstrap/less/theme.less | 0 .../bower_components/bootstrap/less/thumbnails.less | 0 .../bower_components/bootstrap/less/tooltip.less | 0 .../bower_components/bootstrap/less/type.less | 0 .../bower_components/bootstrap/less/utilities.less | 0 .../bower_components/bootstrap/less/variables.less | 0 .../bower_components/bootstrap/less/wells.less | 0 .../bower_components/bootstrap/nuget/MyGet.ps1 | 0 .../bootstrap/nuget/bootstrap.less.nuspec | 0 .../bootstrap/nuget/bootstrap.nuspec | 0 .../static}/bower_components/bootstrap/package.js | 0 .../static}/bower_components/bootstrap/package.json | 0 .../static}/bower_components/clipboard/.bower.json | 0 .../static}/bower_components/clipboard/bower.json | 0 .../bower_components/clipboard/contributing.md | 0 .../bower_components/clipboard/dist/clipboard.js | 0 .../clipboard/dist/clipboard.min.js | 0 .../static}/bower_components/clipboard/package.js | 0 .../static}/bower_components/clipboard/package.json | 0 .../static}/bower_components/clipboard/readme.md | 0 .../bower_components/jquery-color/.bower.json | 0 .../bower_components/jquery-color/.gitignore | 0 .../bower_components/jquery-color/.gitmodules | 0 .../static}/bower_components/jquery-color/.jshintrc | 0 .../bower_components/jquery-color/AUTHORS.TXT | 0 .../bower_components/jquery-color/MIT-LICENSE.txt | 0 .../static}/bower_components/jquery-color/README.md | 0 .../bower_components/jquery-color/color.jquery.json | 0 .../static}/bower_components/jquery-color/grunt.js | 0 .../bower_components/jquery-color/jquery.color.js | 0 .../jquery-color/jquery.color.svg-names.js | 0 .../bower_components/jquery-color/package.json | 0 .../jquery-color/test/data/swarminject.js | 0 .../jquery-color/test/data/testinit.js | 0 .../bower_components/jquery-color/test/index.html | 0 .../jquery-color/test/jquery-1.5.1.js | 0 .../jquery-color/test/jquery-1.5.2.js | 0 .../jquery-color/test/jquery-1.5.js | 0 .../jquery-color/test/jquery-1.6.1.js | 0 .../jquery-color/test/jquery-1.6.2.js | 0 .../jquery-color/test/jquery-1.6.3.js | 0 .../jquery-color/test/jquery-1.6.4.js | 0 .../jquery-color/test/jquery-1.6.js | 0 .../jquery-color/test/jquery-1.7.1.js | 0 .../jquery-color/test/jquery-1.7.2.js | 0 .../jquery-color/test/jquery-1.7.js | 0 .../bower_components/jquery-color/test/jquery.js | 0 .../bower_components/jquery-color/test/test.html | 0 .../jquery-color/test/unit/.jshintrc | 0 .../jquery-color/test/unit/color.js | 0 .../jquery-load-template/.bower.json | 0 .../jquery-load-template/bower.json | 0 .../dist/jquery.loadTemplate.js | 0 .../dist/jquery.loadTemplate.min.js | 0 .../bower_components/jquery-sparkline/.bower.json | 0 .../bower_components/jquery-sparkline/Changelog.txt | 0 .../bower_components/jquery-sparkline/Makefile | 0 .../bower_components/jquery-sparkline/README.md | 0 .../bower_components/jquery-sparkline/bower.json | 0 .../jquery-sparkline/dist/jquery.sparkline.js | 0 .../jquery-sparkline/dist/jquery.sparkline.min.js | 0 .../bower_components/jquery-sparkline/minheader.txt | 0 .../jquery-sparkline/sparkline.jquery.json | 0 .../bower_components/jquery-sparkline/src/base.js | 0 .../jquery-sparkline/src/chart-bar.js | 0 .../jquery-sparkline/src/chart-box.js | 0 .../jquery-sparkline/src/chart-bullet.js | 0 .../jquery-sparkline/src/chart-discrete.js | 0 .../jquery-sparkline/src/chart-line.js | 0 .../jquery-sparkline/src/chart-pie.js | 0 .../jquery-sparkline/src/chart-tristate.js | 0 .../jquery-sparkline/src/defaults.js | 0 .../bower_components/jquery-sparkline/src/footer.js | 0 .../bower_components/jquery-sparkline/src/header.js | 0 .../jquery-sparkline/src/interact.js | 0 .../jquery-sparkline/src/rangemap.js | 0 .../jquery-sparkline/src/simpledraw.js | 0 .../bower_components/jquery-sparkline/src/utils.js | 0 .../jquery-sparkline/src/vcanvas-base.js | 0 .../jquery-sparkline/src/vcanvas-canvas.js | 0 .../jquery-sparkline/src/vcanvas-vml.js | 0 .../bower_components/jquery-sparkline/version.txt | 0 .../static}/bower_components/jquery/.bower.json | 0 .../static}/bower_components/jquery/AUTHORS.txt | 0 .../static}/bower_components/jquery/LICENSE.txt | 0 .../static}/bower_components/jquery/README.md | 0 .../static}/bower_components/jquery/bower.json | 0 .../static}/bower_components/jquery/dist/jquery.js | 0 .../bower_components/jquery/dist/jquery.min.js | 0 .../bower_components/jquery/dist/jquery.min.map | 0 .../jquery/external/sizzle/LICENSE.txt | 0 .../jquery/external/sizzle/dist/sizzle.js | 0 .../jquery/external/sizzle/dist/sizzle.min.js | 0 .../jquery/external/sizzle/dist/sizzle.min.map | 0 .../static}/bower_components/jquery/src/.jshintrc | 0 .../static}/bower_components/jquery/src/ajax.js | 0 .../bower_components/jquery/src/ajax/jsonp.js | 0 .../bower_components/jquery/src/ajax/load.js | 0 .../bower_components/jquery/src/ajax/parseJSON.js | 0 .../bower_components/jquery/src/ajax/parseXML.js | 0 .../bower_components/jquery/src/ajax/script.js | 0 .../jquery/src/ajax/var/location.js | 0 .../bower_components/jquery/src/ajax/var/nonce.js | 0 .../bower_components/jquery/src/ajax/var/rquery.js | 0 .../static}/bower_components/jquery/src/ajax/xhr.js | 0 .../bower_components/jquery/src/attributes.js | 0 .../bower_components/jquery/src/attributes/attr.js | 0 .../jquery/src/attributes/classes.js | 0 .../bower_components/jquery/src/attributes/prop.js | 0 .../jquery/src/attributes/support.js | 0 .../bower_components/jquery/src/attributes/val.js | 0 .../bower_components/jquery/src/callbacks.js | 0 .../static}/bower_components/jquery/src/core.js | 0 .../bower_components/jquery/src/core/access.js | 0 .../bower_components/jquery/src/core/init.js | 0 .../bower_components/jquery/src/core/parseHTML.js | 0 .../bower_components/jquery/src/core/ready.js | 0 .../jquery/src/core/var/rsingleTag.js | 0 .../static}/bower_components/jquery/src/css.js | 0 .../bower_components/jquery/src/css/addGetHookIf.js | 0 .../bower_components/jquery/src/css/adjustCSS.js | 0 .../bower_components/jquery/src/css/curCSS.js | 0 .../jquery/src/css/defaultDisplay.js | 0 .../jquery/src/css/hiddenVisibleSelectors.js | 0 .../bower_components/jquery/src/css/showHide.js | 0 .../bower_components/jquery/src/css/support.js | 0 .../jquery/src/css/var/cssExpand.js | 0 .../jquery/src/css/var/getStyles.js | 0 .../bower_components/jquery/src/css/var/isHidden.js | 0 .../bower_components/jquery/src/css/var/rmargin.js | 0 .../jquery/src/css/var/rnumnonpx.js | 0 .../bower_components/jquery/src/css/var/swap.js | 0 .../static}/bower_components/jquery/src/data.js | 0 .../bower_components/jquery/src/data/Data.js | 0 .../jquery/src/data/var/acceptData.js | 0 .../jquery/src/data/var/dataPriv.js | 0 .../jquery/src/data/var/dataUser.js | 0 .../static}/bower_components/jquery/src/deferred.js | 0 .../bower_components/jquery/src/deprecated.js | 0 .../bower_components/jquery/src/dimensions.js | 0 .../static}/bower_components/jquery/src/effects.js | 0 .../bower_components/jquery/src/effects/Tween.js | 0 .../jquery/src/effects/animatedSelector.js | 0 .../static}/bower_components/jquery/src/event.js | 0 .../bower_components/jquery/src/event/ajax.js | 0 .../bower_components/jquery/src/event/alias.js | 0 .../bower_components/jquery/src/event/focusin.js | 0 .../bower_components/jquery/src/event/support.js | 0 .../bower_components/jquery/src/event/trigger.js | 0 .../bower_components/jquery/src/exports/amd.js | 0 .../bower_components/jquery/src/exports/global.js | 0 .../static}/bower_components/jquery/src/intro.js | 0 .../static}/bower_components/jquery/src/jquery.js | 0 .../bower_components/jquery/src/manipulation.js | 0 .../jquery/src/manipulation/_evalUrl.js | 0 .../jquery/src/manipulation/buildFragment.js | 0 .../jquery/src/manipulation/getAll.js | 0 .../jquery/src/manipulation/setGlobalEval.js | 0 .../jquery/src/manipulation/support.js | 0 .../jquery/src/manipulation/var/rcheckableType.js | 0 .../jquery/src/manipulation/var/rscriptType.js | 0 .../jquery/src/manipulation/var/rtagName.js | 0 .../jquery/src/manipulation/wrapMap.js | 0 .../static}/bower_components/jquery/src/offset.js | 0 .../static}/bower_components/jquery/src/outro.js | 0 .../static}/bower_components/jquery/src/queue.js | 0 .../bower_components/jquery/src/queue/delay.js | 0 .../bower_components/jquery/src/selector-native.js | 0 .../bower_components/jquery/src/selector-sizzle.js | 0 .../static}/bower_components/jquery/src/selector.js | 0 .../bower_components/jquery/src/serialize.js | 0 .../bower_components/jquery/src/traversing.js | 0 .../jquery/src/traversing/findFilter.js | 0 .../jquery/src/traversing/var/dir.js | 0 .../jquery/src/traversing/var/rneedsContext.js | 0 .../jquery/src/traversing/var/siblings.js | 0 .../static}/bower_components/jquery/src/var/arr.js | 0 .../bower_components/jquery/src/var/class2type.js | 0 .../bower_components/jquery/src/var/concat.js | 0 .../bower_components/jquery/src/var/document.js | 0 .../jquery/src/var/documentElement.js | 0 .../bower_components/jquery/src/var/hasOwn.js | 0 .../bower_components/jquery/src/var/indexOf.js | 0 .../static}/bower_components/jquery/src/var/pnum.js | 0 .../static}/bower_components/jquery/src/var/push.js | 0 .../bower_components/jquery/src/var/rcssNum.js | 0 .../bower_components/jquery/src/var/rnotwhite.js | 0 .../bower_components/jquery/src/var/slice.js | 0 .../bower_components/jquery/src/var/support.js | 0 .../bower_components/jquery/src/var/toString.js | 0 .../static}/bower_components/jquery/src/wrap.js | 0 .../static}/bower_components/moment/.bower.json | 0 .../static}/bower_components/moment/CHANGELOG.md | 0 .../static}/bower_components/moment/LICENSE | 0 .../static}/bower_components/moment/README.md | 0 .../static}/bower_components/moment/bower.json | 0 .../static}/bower_components/moment/locale/af.js | 0 .../static}/bower_components/moment/locale/ar-dz.js | 0 .../static}/bower_components/moment/locale/ar-ly.js | 0 .../static}/bower_components/moment/locale/ar-ma.js | 0 .../static}/bower_components/moment/locale/ar-sa.js | 0 .../static}/bower_components/moment/locale/ar-tn.js | 0 .../static}/bower_components/moment/locale/ar.js | 0 .../static}/bower_components/moment/locale/az.js | 0 .../static}/bower_components/moment/locale/be.js | 0 .../static}/bower_components/moment/locale/bg.js | 0 .../static}/bower_components/moment/locale/bn.js | 0 .../static}/bower_components/moment/locale/bo.js | 0 .../static}/bower_components/moment/locale/br.js | 0 .../static}/bower_components/moment/locale/bs.js | 0 .../static}/bower_components/moment/locale/ca.js | 0 .../static}/bower_components/moment/locale/cs.js | 0 .../static}/bower_components/moment/locale/cv.js | 0 .../static}/bower_components/moment/locale/cy.js | 0 .../static}/bower_components/moment/locale/da.js | 0 .../static}/bower_components/moment/locale/de-at.js | 0 .../static}/bower_components/moment/locale/de.js | 0 .../static}/bower_components/moment/locale/dv.js | 0 .../static}/bower_components/moment/locale/el.js | 0 .../static}/bower_components/moment/locale/en-au.js | 0 .../static}/bower_components/moment/locale/en-ca.js | 0 .../static}/bower_components/moment/locale/en-gb.js | 0 .../static}/bower_components/moment/locale/en-ie.js | 0 .../static}/bower_components/moment/locale/en-nz.js | 0 .../static}/bower_components/moment/locale/eo.js | 0 .../static}/bower_components/moment/locale/es-do.js | 0 .../static}/bower_components/moment/locale/es.js | 0 .../static}/bower_components/moment/locale/et.js | 0 .../static}/bower_components/moment/locale/eu.js | 0 .../static}/bower_components/moment/locale/fa.js | 0 .../static}/bower_components/moment/locale/fi.js | 0 .../static}/bower_components/moment/locale/fo.js | 0 .../static}/bower_components/moment/locale/fr-ca.js | 0 .../static}/bower_components/moment/locale/fr-ch.js | 0 .../static}/bower_components/moment/locale/fr.js | 0 .../static}/bower_components/moment/locale/fy.js | 0 .../static}/bower_components/moment/locale/gd.js | 0 .../static}/bower_components/moment/locale/gl.js | 0 .../static}/bower_components/moment/locale/he.js | 0 .../static}/bower_components/moment/locale/hi.js | 0 .../static}/bower_components/moment/locale/hr.js | 0 .../static}/bower_components/moment/locale/hu.js | 0 .../static}/bower_components/moment/locale/hy-am.js | 0 .../static}/bower_components/moment/locale/id.js | 0 .../static}/bower_components/moment/locale/is.js | 0 .../static}/bower_components/moment/locale/it.js | 0 .../static}/bower_components/moment/locale/ja.js | 0 .../static}/bower_components/moment/locale/jv.js | 0 .../static}/bower_components/moment/locale/ka.js | 0 .../static}/bower_components/moment/locale/kk.js | 0 .../static}/bower_components/moment/locale/km.js | 0 .../static}/bower_components/moment/locale/ko.js | 0 .../static}/bower_components/moment/locale/ky.js | 0 .../static}/bower_components/moment/locale/lb.js | 0 .../static}/bower_components/moment/locale/lo.js | 0 .../static}/bower_components/moment/locale/lt.js | 0 .../static}/bower_components/moment/locale/lv.js | 0 .../static}/bower_components/moment/locale/me.js | 0 .../static}/bower_components/moment/locale/mi.js | 0 .../static}/bower_components/moment/locale/mk.js | 0 .../static}/bower_components/moment/locale/ml.js | 0 .../static}/bower_components/moment/locale/mr.js | 0 .../static}/bower_components/moment/locale/ms-my.js | 0 .../static}/bower_components/moment/locale/ms.js | 0 .../static}/bower_components/moment/locale/my.js | 0 .../static}/bower_components/moment/locale/nb.js | 0 .../static}/bower_components/moment/locale/ne.js | 0 .../static}/bower_components/moment/locale/nl-be.js | 0 .../static}/bower_components/moment/locale/nl.js | 0 .../static}/bower_components/moment/locale/nn.js | 0 .../static}/bower_components/moment/locale/pa-in.js | 0 .../static}/bower_components/moment/locale/pl.js | 0 .../static}/bower_components/moment/locale/pt-br.js | 0 .../static}/bower_components/moment/locale/pt.js | 0 .../static}/bower_components/moment/locale/ro.js | 0 .../static}/bower_components/moment/locale/ru.js | 0 .../static}/bower_components/moment/locale/se.js | 0 .../static}/bower_components/moment/locale/si.js | 0 .../static}/bower_components/moment/locale/sk.js | 0 .../static}/bower_components/moment/locale/sl.js | 0 .../static}/bower_components/moment/locale/sq.js | 0 .../bower_components/moment/locale/sr-cyrl.js | 0 .../static}/bower_components/moment/locale/sr.js | 0 .../static}/bower_components/moment/locale/ss.js | 0 .../static}/bower_components/moment/locale/sv.js | 0 .../static}/bower_components/moment/locale/sw.js | 0 .../static}/bower_components/moment/locale/ta.js | 0 .../static}/bower_components/moment/locale/te.js | 0 .../static}/bower_components/moment/locale/tet.js | 0 .../static}/bower_components/moment/locale/th.js | 0 .../static}/bower_components/moment/locale/tl-ph.js | 0 .../static}/bower_components/moment/locale/tlh.js | 0 .../static}/bower_components/moment/locale/tr.js | 0 .../static}/bower_components/moment/locale/tzl.js | 0 .../bower_components/moment/locale/tzm-latn.js | 0 .../static}/bower_components/moment/locale/tzm.js | 0 .../static}/bower_components/moment/locale/uk.js | 0 .../static}/bower_components/moment/locale/uz.js | 0 .../static}/bower_components/moment/locale/vi.js | 0 .../bower_components/moment/locale/x-pseudo.js | 0 .../static}/bower_components/moment/locale/yo.js | 0 .../static}/bower_components/moment/locale/zh-cn.js | 0 .../static}/bower_components/moment/locale/zh-hk.js | 0 .../static}/bower_components/moment/locale/zh-tw.js | 0 .../static}/bower_components/moment/min/locales.js | 0 .../bower_components/moment/min/locales.min.js | 0 .../moment/min/moment-with-locales.js | 0 .../moment/min/moment-with-locales.min.js | 0 .../bower_components/moment/min/moment.min.js | 0 .../static}/bower_components/moment/min/tests.js | 0 .../static}/bower_components/moment/moment.d.ts | 0 .../static}/bower_components/moment/moment.js | 0 .../moment/src/lib/create/check-overflow.js | 0 .../moment/src/lib/create/date-from-array.js | 0 .../moment/src/lib/create/from-anything.js | 0 .../moment/src/lib/create/from-array.js | 0 .../moment/src/lib/create/from-object.js | 0 .../moment/src/lib/create/from-string-and-array.js | 0 .../moment/src/lib/create/from-string-and-format.js | 0 .../moment/src/lib/create/from-string.js | 0 .../bower_components/moment/src/lib/create/local.js | 0 .../moment/src/lib/create/parsing-flags.js | 0 .../bower_components/moment/src/lib/create/utc.js | 0 .../bower_components/moment/src/lib/create/valid.js | 0 .../bower_components/moment/src/lib/duration/abs.js | 0 .../moment/src/lib/duration/add-subtract.js | 0 .../bower_components/moment/src/lib/duration/as.js | 0 .../moment/src/lib/duration/bubble.js | 0 .../moment/src/lib/duration/constructor.js | 0 .../moment/src/lib/duration/create.js | 0 .../moment/src/lib/duration/duration.js | 0 .../bower_components/moment/src/lib/duration/get.js | 0 .../moment/src/lib/duration/humanize.js | 0 .../moment/src/lib/duration/iso-string.js | 0 .../moment/src/lib/duration/prototype.js | 0 .../moment/src/lib/format/format.js | 0 .../moment/src/lib/locale/base-config.js | 0 .../moment/src/lib/locale/calendar.js | 0 .../moment/src/lib/locale/constructor.js | 0 .../bower_components/moment/src/lib/locale/en.js | 0 .../moment/src/lib/locale/formats.js | 0 .../moment/src/lib/locale/invalid.js | 0 .../bower_components/moment/src/lib/locale/lists.js | 0 .../moment/src/lib/locale/locale.js | 0 .../moment/src/lib/locale/locales.js | 0 .../moment/src/lib/locale/ordinal.js | 0 .../moment/src/lib/locale/pre-post-format.js | 0 .../moment/src/lib/locale/prototype.js | 0 .../moment/src/lib/locale/relative.js | 0 .../bower_components/moment/src/lib/locale/set.js | 0 .../moment/src/lib/moment/add-subtract.js | 0 .../moment/src/lib/moment/calendar.js | 0 .../bower_components/moment/src/lib/moment/clone.js | 0 .../moment/src/lib/moment/compare.js | 0 .../moment/src/lib/moment/constructor.js | 0 .../moment/src/lib/moment/creation-data.js | 0 .../bower_components/moment/src/lib/moment/diff.js | 0 .../moment/src/lib/moment/format.js | 0 .../bower_components/moment/src/lib/moment/from.js | 0 .../moment/src/lib/moment/get-set.js | 0 .../moment/src/lib/moment/locale.js | 0 .../moment/src/lib/moment/min-max.js | 0 .../moment/src/lib/moment/moment.js | 0 .../bower_components/moment/src/lib/moment/now.js | 0 .../moment/src/lib/moment/prototype.js | 0 .../moment/src/lib/moment/start-end-of.js | 0 .../moment/src/lib/moment/to-type.js | 0 .../bower_components/moment/src/lib/moment/to.js | 0 .../bower_components/moment/src/lib/moment/valid.js | 0 .../bower_components/moment/src/lib/parse/regex.js | 0 .../bower_components/moment/src/lib/parse/token.js | 0 .../moment/src/lib/units/aliases.js | 0 .../moment/src/lib/units/constants.js | 0 .../moment/src/lib/units/day-of-month.js | 0 .../moment/src/lib/units/day-of-week.js | 0 .../moment/src/lib/units/day-of-year.js | 0 .../bower_components/moment/src/lib/units/hour.js | 0 .../moment/src/lib/units/millisecond.js | 0 .../bower_components/moment/src/lib/units/minute.js | 0 .../bower_components/moment/src/lib/units/month.js | 0 .../bower_components/moment/src/lib/units/offset.js | 0 .../moment/src/lib/units/priorities.js | 0 .../moment/src/lib/units/quarter.js | 0 .../bower_components/moment/src/lib/units/second.js | 0 .../moment/src/lib/units/timestamp.js | 0 .../moment/src/lib/units/timezone.js | 0 .../bower_components/moment/src/lib/units/units.js | 0 .../moment/src/lib/units/week-calendar-utils.js | 0 .../moment/src/lib/units/week-year.js | 0 .../bower_components/moment/src/lib/units/week.js | 0 .../bower_components/moment/src/lib/units/year.js | 0 .../moment/src/lib/utils/abs-ceil.js | 0 .../moment/src/lib/utils/abs-floor.js | 0 .../moment/src/lib/utils/abs-round.js | 0 .../moment/src/lib/utils/compare-arrays.js | 0 .../moment/src/lib/utils/defaults.js | 0 .../moment/src/lib/utils/deprecate.js | 0 .../bower_components/moment/src/lib/utils/extend.js | 0 .../moment/src/lib/utils/has-own-prop.js | 0 .../bower_components/moment/src/lib/utils/hooks.js | 0 .../moment/src/lib/utils/index-of.js | 0 .../moment/src/lib/utils/is-array.js | 0 .../moment/src/lib/utils/is-date.js | 0 .../moment/src/lib/utils/is-function.js | 0 .../moment/src/lib/utils/is-number.js | 0 .../moment/src/lib/utils/is-object-empty.js | 0 .../moment/src/lib/utils/is-object.js | 0 .../moment/src/lib/utils/is-undefined.js | 0 .../bower_components/moment/src/lib/utils/keys.js | 0 .../bower_components/moment/src/lib/utils/map.js | 0 .../bower_components/moment/src/lib/utils/some.js | 0 .../bower_components/moment/src/lib/utils/to-int.js | 0 .../moment/src/lib/utils/zero-fill.js | 0 .../bower_components/moment/src/locale/af.js | 0 .../bower_components/moment/src/locale/ar-dz.js | 0 .../bower_components/moment/src/locale/ar-ly.js | 0 .../bower_components/moment/src/locale/ar-ma.js | 0 .../bower_components/moment/src/locale/ar-sa.js | 0 .../bower_components/moment/src/locale/ar-tn.js | 0 .../bower_components/moment/src/locale/ar.js | 0 .../bower_components/moment/src/locale/az.js | 0 .../bower_components/moment/src/locale/be.js | 0 .../bower_components/moment/src/locale/bg.js | 0 .../bower_components/moment/src/locale/bn.js | 0 .../bower_components/moment/src/locale/bo.js | 0 .../bower_components/moment/src/locale/br.js | 0 .../bower_components/moment/src/locale/bs.js | 0 .../bower_components/moment/src/locale/ca.js | 0 .../bower_components/moment/src/locale/cs.js | 0 .../bower_components/moment/src/locale/cv.js | 0 .../bower_components/moment/src/locale/cy.js | 0 .../bower_components/moment/src/locale/da.js | 0 .../bower_components/moment/src/locale/de-at.js | 0 .../bower_components/moment/src/locale/de.js | 0 .../bower_components/moment/src/locale/dv.js | 0 .../bower_components/moment/src/locale/el.js | 0 .../bower_components/moment/src/locale/en-au.js | 0 .../bower_components/moment/src/locale/en-ca.js | 0 .../bower_components/moment/src/locale/en-gb.js | 0 .../bower_components/moment/src/locale/en-ie.js | 0 .../bower_components/moment/src/locale/en-nz.js | 0 .../bower_components/moment/src/locale/eo.js | 0 .../bower_components/moment/src/locale/es-do.js | 0 .../bower_components/moment/src/locale/es.js | 0 .../bower_components/moment/src/locale/et.js | 0 .../bower_components/moment/src/locale/eu.js | 0 .../bower_components/moment/src/locale/fa.js | 0 .../bower_components/moment/src/locale/fi.js | 0 .../bower_components/moment/src/locale/fo.js | 0 .../bower_components/moment/src/locale/fr-ca.js | 0 .../bower_components/moment/src/locale/fr-ch.js | 0 .../bower_components/moment/src/locale/fr.js | 0 .../bower_components/moment/src/locale/fy.js | 0 .../bower_components/moment/src/locale/gd.js | 0 .../bower_components/moment/src/locale/gl.js | 0 .../bower_components/moment/src/locale/he.js | 0 .../bower_components/moment/src/locale/hi.js | 0 .../bower_components/moment/src/locale/hr.js | 0 .../bower_components/moment/src/locale/hu.js | 0 .../bower_components/moment/src/locale/hy-am.js | 0 .../bower_components/moment/src/locale/id.js | 0 .../bower_components/moment/src/locale/is.js | 0 .../bower_components/moment/src/locale/it.js | 0 .../bower_components/moment/src/locale/ja.js | 0 .../bower_components/moment/src/locale/jv.js | 0 .../bower_components/moment/src/locale/ka.js | 0 .../bower_components/moment/src/locale/kk.js | 0 .../bower_components/moment/src/locale/km.js | 0 .../bower_components/moment/src/locale/ko.js | 0 .../bower_components/moment/src/locale/ky.js | 0 .../bower_components/moment/src/locale/lb.js | 0 .../bower_components/moment/src/locale/lo.js | 0 .../bower_components/moment/src/locale/lt.js | 0 .../bower_components/moment/src/locale/lv.js | 0 .../bower_components/moment/src/locale/me.js | 0 .../bower_components/moment/src/locale/mi.js | 0 .../bower_components/moment/src/locale/mk.js | 0 .../bower_components/moment/src/locale/ml.js | 0 .../bower_components/moment/src/locale/mr.js | 0 .../bower_components/moment/src/locale/ms-my.js | 0 .../bower_components/moment/src/locale/ms.js | 0 .../bower_components/moment/src/locale/my.js | 0 .../bower_components/moment/src/locale/nb.js | 0 .../bower_components/moment/src/locale/ne.js | 0 .../bower_components/moment/src/locale/nl-be.js | 0 .../bower_components/moment/src/locale/nl.js | 0 .../bower_components/moment/src/locale/nn.js | 0 .../bower_components/moment/src/locale/pa-in.js | 0 .../bower_components/moment/src/locale/pl.js | 0 .../bower_components/moment/src/locale/pt-br.js | 0 .../bower_components/moment/src/locale/pt.js | 0 .../bower_components/moment/src/locale/ro.js | 0 .../bower_components/moment/src/locale/ru.js | 0 .../bower_components/moment/src/locale/se.js | 0 .../bower_components/moment/src/locale/si.js | 0 .../bower_components/moment/src/locale/sk.js | 0 .../bower_components/moment/src/locale/sl.js | 0 .../bower_components/moment/src/locale/sq.js | 0 .../bower_components/moment/src/locale/sr-cyrl.js | 0 .../bower_components/moment/src/locale/sr.js | 0 .../bower_components/moment/src/locale/ss.js | 0 .../bower_components/moment/src/locale/sv.js | 0 .../bower_components/moment/src/locale/sw.js | 0 .../bower_components/moment/src/locale/ta.js | 0 .../bower_components/moment/src/locale/te.js | 0 .../bower_components/moment/src/locale/tet.js | 0 .../bower_components/moment/src/locale/th.js | 0 .../bower_components/moment/src/locale/tl-ph.js | 0 .../bower_components/moment/src/locale/tlh.js | 0 .../bower_components/moment/src/locale/tr.js | 0 .../bower_components/moment/src/locale/tzl.js | 0 .../bower_components/moment/src/locale/tzm-latn.js | 0 .../bower_components/moment/src/locale/tzm.js | 0 .../bower_components/moment/src/locale/uk.js | 0 .../bower_components/moment/src/locale/uz.js | 0 .../bower_components/moment/src/locale/vi.js | 0 .../bower_components/moment/src/locale/x-pseudo.js | 0 .../bower_components/moment/src/locale/yo.js | 0 .../bower_components/moment/src/locale/zh-cn.js | 0 .../bower_components/moment/src/locale/zh-hk.js | 0 .../bower_components/moment/src/locale/zh-tw.js | 0 .../static}/bower_components/moment/src/moment.js | 0 .../bower_components/moment/templates/default.js | 0 .../moment/templates/locale-header.js | 0 .../moment/templates/test-header.js | 0 {themes/bootstrap/public => ui/static}/favicon.png | Bin {themes/bootstrap/public => ui/static}/inbucket.css | 0 {themes/bootstrap/public => ui/static}/mailbox.js | 0 {themes/bootstrap/public => ui/static}/metrics.js | 0 {themes/bootstrap/public => ui/static}/monitor.js | 0 {themes/bootstrap => ui}/templates/_base.html | 0 .../bootstrap => ui}/templates/mailbox/_html.html | 0 .../bootstrap => ui}/templates/mailbox/_show.html | 0 .../bootstrap => ui}/templates/mailbox/index.html | 0 {themes/bootstrap => ui}/templates/root/index.html | 0 .../bootstrap => ui}/templates/root/monitor.html | 0 {themes/bootstrap => ui}/templates/root/status.html | 0 660 files changed, 5 insertions(+), 3 deletions(-) rename {themes => ui}/greeting.html (100%) rename {themes/bootstrap/public => ui/static}/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/CHANGELOG.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/Gemfile (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/Gemfile.lock (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/Gruntfile.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/ISSUE_TEMPLATE.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/LICENSE (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/README.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap-theme.css (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap-theme.css.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap-theme.min.css (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap-theme.min.css.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap.css (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap.css.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap.min.css (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/css/bootstrap.min.css.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.eot (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.svg (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.ttf (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/js/bootstrap.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/js/bootstrap.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/dist/js/npm.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/fonts/glyphicons-halflings-regular.eot (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/fonts/glyphicons-halflings-regular.svg (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/fonts/glyphicons-halflings-regular.ttf (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff2 (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/.jshintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/bs-commonjs-generator.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/bs-glyphicons-data-generator.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/bs-lessdoc-parser.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/bs-raw-files-generator.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/change-version.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/configBridge.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/npm-shrinkwrap.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/grunt/sauce_browsers.yml (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/.jscsrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/.jshintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/affix.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/alert.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/button.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/carousel.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/collapse.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/dropdown.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/modal.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/popover.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/scrollspy.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/tab.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/tooltip.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/js/transition.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/.csscomb.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/.csslintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/alerts.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/badges.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/bootstrap.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/breadcrumbs.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/button-groups.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/buttons.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/carousel.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/close.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/code.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/component-animations.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/dropdowns.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/forms.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/glyphicons.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/grid.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/input-groups.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/jumbotron.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/labels.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/list-group.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/media.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/alerts.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/background-variant.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/border-radius.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/buttons.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/center-block.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/clearfix.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/forms.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/gradients.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/grid-framework.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/grid.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/hide-text.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/image.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/labels.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/list-group.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/nav-divider.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/nav-vertical-align.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/opacity.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/pagination.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/panels.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/progress-bar.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/reset-filter.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/reset-text.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/resize.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/responsive-visibility.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/size.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/tab-focus.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/table-row.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/text-emphasis.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/text-overflow.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/mixins/vendor-prefixes.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/modals.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/navbar.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/navs.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/normalize.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/pager.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/pagination.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/panels.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/popovers.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/print.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/progress-bars.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/responsive-embed.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/responsive-utilities.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/scaffolding.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/tables.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/theme.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/thumbnails.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/tooltip.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/type.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/utilities.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/variables.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/less/wells.less (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/nuget/MyGet.ps1 (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/nuget/bootstrap.less.nuspec (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/nuget/bootstrap.nuspec (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/package.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/bootstrap/package.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/contributing.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/dist/clipboard.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/dist/clipboard.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/package.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/package.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/clipboard/readme.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/.gitignore (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/.gitmodules (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/.jshintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/AUTHORS.TXT (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/MIT-LICENSE.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/README.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/color.jquery.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/grunt.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/jquery.color.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/jquery.color.svg-names.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/package.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/data/swarminject.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/data/testinit.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/index.html (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.5.1.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.5.2.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.5.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.6.1.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.6.2.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.6.3.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.6.4.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.6.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.7.1.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.7.2.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery-1.7.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/jquery.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/test.html (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/unit/.jshintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-color/test/unit/color.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-load-template/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-load-template/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-load-template/dist/jquery.loadTemplate.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-load-template/dist/jquery.loadTemplate.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/Changelog.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/Makefile (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/README.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/dist/jquery.sparkline.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/dist/jquery.sparkline.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/minheader.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/sparkline.jquery.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/base.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-bar.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-box.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-bullet.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-discrete.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-line.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-pie.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/chart-tristate.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/defaults.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/footer.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/header.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/interact.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/rangemap.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/simpledraw.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/utils.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/vcanvas-base.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/vcanvas-canvas.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/src/vcanvas-vml.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery-sparkline/version.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/AUTHORS.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/LICENSE.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/README.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/dist/jquery.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/dist/jquery.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/dist/jquery.min.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/external/sizzle/LICENSE.txt (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/external/sizzle/dist/sizzle.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/external/sizzle/dist/sizzle.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/external/sizzle/dist/sizzle.min.map (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/.jshintrc (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/jsonp.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/load.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/parseJSON.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/parseXML.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/script.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/var/location.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/var/nonce.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/var/rquery.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/ajax/xhr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes/attr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes/classes.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes/prop.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes/support.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/attributes/val.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/callbacks.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core/access.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core/init.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core/parseHTML.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core/ready.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/core/var/rsingleTag.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/addGetHookIf.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/adjustCSS.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/curCSS.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/defaultDisplay.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/hiddenVisibleSelectors.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/showHide.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/support.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/cssExpand.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/getStyles.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/isHidden.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/rmargin.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/rnumnonpx.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/css/var/swap.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/data.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/data/Data.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/data/var/acceptData.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/data/var/dataPriv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/data/var/dataUser.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/deferred.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/deprecated.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/dimensions.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/effects.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/effects/Tween.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/effects/animatedSelector.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event/ajax.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event/alias.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event/focusin.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event/support.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/event/trigger.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/exports/amd.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/exports/global.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/intro.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/jquery.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/_evalUrl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/buildFragment.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/getAll.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/setGlobalEval.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/support.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/var/rcheckableType.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/var/rscriptType.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/var/rtagName.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/manipulation/wrapMap.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/offset.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/outro.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/queue.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/queue/delay.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/selector-native.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/selector-sizzle.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/selector.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/serialize.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/traversing.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/traversing/findFilter.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/traversing/var/dir.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/traversing/var/rneedsContext.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/traversing/var/siblings.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/arr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/class2type.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/concat.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/document.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/documentElement.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/hasOwn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/indexOf.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/pnum.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/push.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/rcssNum.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/rnotwhite.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/slice.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/support.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/var/toString.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/jquery/src/wrap.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/.bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/CHANGELOG.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/LICENSE (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/README.md (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/bower.json (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/af.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar-dz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar-ly.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar-ma.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar-sa.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar-tn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ar.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/az.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/be.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/bg.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/bn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/bo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/br.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/bs.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/cs.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/cv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/cy.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/da.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/de-at.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/de.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/dv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/el.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/en-au.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/en-ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/en-gb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/en-ie.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/en-nz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/eo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/es-do.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/es.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/et.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/eu.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fa.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fr-ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fr-ch.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/fy.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/gd.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/gl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/he.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/hi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/hr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/hu.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/hy-am.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/id.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/is.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/it.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ja.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/jv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ka.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/kk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/km.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ko.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ky.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/lb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/lo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/lt.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/lv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/me.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/mi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/mk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ml.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/mr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ms-my.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ms.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/my.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/nb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ne.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/nl-be.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/nl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/nn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/pa-in.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/pl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/pt-br.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/pt.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ro.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ru.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/se.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/si.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sq.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sr-cyrl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ss.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/sw.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/ta.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/te.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tet.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/th.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tl-ph.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tlh.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tzl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tzm-latn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/tzm.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/uk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/uz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/vi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/x-pseudo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/yo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/zh-cn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/zh-hk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/locale/zh-tw.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/locales.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/locales.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/moment-with-locales.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/moment-with-locales.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/moment.min.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/min/tests.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/moment.d.ts (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/moment.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/check-overflow.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/date-from-array.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-anything.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-array.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-object.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-string-and-array.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-string-and-format.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/from-string.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/local.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/parsing-flags.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/utc.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/create/valid.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/abs.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/add-subtract.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/as.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/bubble.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/constructor.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/create.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/duration.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/get.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/humanize.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/iso-string.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/duration/prototype.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/format/format.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/base-config.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/calendar.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/constructor.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/en.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/formats.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/invalid.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/lists.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/locale.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/locales.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/ordinal.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/pre-post-format.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/prototype.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/relative.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/locale/set.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/add-subtract.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/calendar.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/clone.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/compare.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/constructor.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/creation-data.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/diff.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/format.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/from.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/get-set.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/locale.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/min-max.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/moment.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/now.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/prototype.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/start-end-of.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/to-type.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/to.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/moment/valid.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/parse/regex.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/parse/token.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/aliases.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/constants.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/day-of-month.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/day-of-week.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/day-of-year.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/hour.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/millisecond.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/minute.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/month.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/offset.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/priorities.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/quarter.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/second.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/timestamp.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/timezone.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/units.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/week-calendar-utils.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/week-year.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/week.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/units/year.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/abs-ceil.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/abs-floor.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/abs-round.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/compare-arrays.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/defaults.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/deprecate.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/extend.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/has-own-prop.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/hooks.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/index-of.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-array.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-date.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-function.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-number.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-object-empty.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-object.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/is-undefined.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/keys.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/map.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/some.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/to-int.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/lib/utils/zero-fill.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/af.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar-dz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar-ly.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar-ma.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar-sa.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar-tn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ar.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/az.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/be.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/bg.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/bn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/bo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/br.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/bs.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/cs.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/cv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/cy.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/da.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/de-at.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/de.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/dv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/el.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/en-au.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/en-ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/en-gb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/en-ie.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/en-nz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/eo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/es-do.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/es.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/et.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/eu.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fa.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fr-ca.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fr-ch.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/fy.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/gd.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/gl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/he.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/hi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/hr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/hu.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/hy-am.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/id.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/is.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/it.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ja.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/jv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ka.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/kk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/km.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ko.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ky.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/lb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/lo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/lt.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/lv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/me.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/mi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/mk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ml.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/mr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ms-my.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ms.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/my.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/nb.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ne.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/nl-be.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/nl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/nn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/pa-in.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/pl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/pt-br.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/pt.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ro.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ru.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/se.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/si.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sq.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sr-cyrl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ss.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sv.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/sw.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/ta.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/te.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tet.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/th.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tl-ph.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tlh.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tr.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tzl.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tzm-latn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/tzm.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/uk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/uz.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/vi.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/x-pseudo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/yo.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/zh-cn.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/zh-hk.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/locale/zh-tw.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/src/moment.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/templates/default.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/templates/locale-header.js (100%) rename {themes/bootstrap/public => ui/static}/bower_components/moment/templates/test-header.js (100%) rename {themes/bootstrap/public => ui/static}/favicon.png (100%) rename {themes/bootstrap/public => ui/static}/inbucket.css (100%) rename {themes/bootstrap/public => ui/static}/mailbox.js (100%) rename {themes/bootstrap/public => ui/static}/metrics.js (100%) rename {themes/bootstrap/public => ui/static}/monitor.js (100%) rename {themes/bootstrap => ui}/templates/_base.html (100%) rename {themes/bootstrap => ui}/templates/mailbox/_html.html (100%) rename {themes/bootstrap => ui}/templates/mailbox/_show.html (100%) rename {themes/bootstrap => ui}/templates/mailbox/index.html (100%) rename {themes/bootstrap => ui}/templates/root/index.html (100%) rename {themes/bootstrap => ui}/templates/root/monitor.html (100%) rename {themes/bootstrap => ui}/templates/root/status.html (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c226d2..1aba3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Changed - Massive refactor of back-end code. Inbucket should now be both easier and more enjoyable to work on. +- Renamed `themes` directory to `ui` and eliminated the intermediate `bootstrap` + directory. ## [v1.3.1] - 2018-03-10 diff --git a/pkg/config/config.go b/pkg/config/config.go index 24017f4..2ca6702 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -57,10 +57,10 @@ type POP3 struct { // Web contains the HTTP server configuration. type Web struct { Addr string `required:"true" default:"0.0.0.0:9000" desc:"Web server IP4 host:port"` - TemplateDir string `required:"true" default:"themes/bootstrap/templates" desc:"Theme template dir"` + TemplateDir string `required:"true" default:"ui/templates" desc:"Theme template dir"` TemplateCache bool `required:"true" default:"true" desc:"Cache templates after first use?"` - PublicDir string `required:"true" default:"themes/bootstrap/public" desc:"Theme public dir"` - GreetingFile string `required:"true" default:"themes/greeting.html" desc:"Home page greeting HTML"` + PublicDir string `required:"true" default:"ui/static" desc:"Theme public dir"` + GreetingFile string `required:"true" default:"ui/greeting.html" desc:"Home page greeting HTML"` MailboxPrompt string `required:"true" default:"@inbucket" desc:"Prompt next to mailbox input"` CookieAuthKey string `desc:"Session cipher key (text)"` MonitorVisible bool `required:"true" default:"true" desc:"Show monitor tab in UI?"` diff --git a/themes/greeting.html b/ui/greeting.html similarity index 100% rename from themes/greeting.html rename to ui/greeting.html diff --git a/themes/bootstrap/public/bower.json b/ui/static/bower.json similarity index 100% rename from themes/bootstrap/public/bower.json rename to ui/static/bower.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/.bower.json b/ui/static/bower_components/bootstrap/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/.bower.json rename to ui/static/bower_components/bootstrap/.bower.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/CHANGELOG.md b/ui/static/bower_components/bootstrap/CHANGELOG.md similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/CHANGELOG.md rename to ui/static/bower_components/bootstrap/CHANGELOG.md diff --git a/themes/bootstrap/public/bower_components/bootstrap/Gemfile b/ui/static/bower_components/bootstrap/Gemfile similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/Gemfile rename to ui/static/bower_components/bootstrap/Gemfile diff --git a/themes/bootstrap/public/bower_components/bootstrap/Gemfile.lock b/ui/static/bower_components/bootstrap/Gemfile.lock similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/Gemfile.lock rename to ui/static/bower_components/bootstrap/Gemfile.lock diff --git a/themes/bootstrap/public/bower_components/bootstrap/Gruntfile.js b/ui/static/bower_components/bootstrap/Gruntfile.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/Gruntfile.js rename to ui/static/bower_components/bootstrap/Gruntfile.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/ISSUE_TEMPLATE.md b/ui/static/bower_components/bootstrap/ISSUE_TEMPLATE.md similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/ISSUE_TEMPLATE.md rename to ui/static/bower_components/bootstrap/ISSUE_TEMPLATE.md diff --git a/themes/bootstrap/public/bower_components/bootstrap/LICENSE b/ui/static/bower_components/bootstrap/LICENSE similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/LICENSE rename to ui/static/bower_components/bootstrap/LICENSE diff --git a/themes/bootstrap/public/bower_components/bootstrap/README.md b/ui/static/bower_components/bootstrap/README.md similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/README.md rename to ui/static/bower_components/bootstrap/README.md diff --git a/themes/bootstrap/public/bower_components/bootstrap/bower.json b/ui/static/bower_components/bootstrap/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/bower.json rename to ui/static/bower_components/bootstrap/bower.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.css b/ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.css similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.css rename to ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.css diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.css.map b/ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.css.map similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.css.map rename to ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.css.map diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.min.css b/ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.min.css similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.min.css rename to ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.min.css diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.min.css.map b/ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.min.css.map similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap-theme.min.css.map rename to ui/static/bower_components/bootstrap/dist/css/bootstrap-theme.min.css.map diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.css b/ui/static/bower_components/bootstrap/dist/css/bootstrap.css similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.css rename to ui/static/bower_components/bootstrap/dist/css/bootstrap.css diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.css.map b/ui/static/bower_components/bootstrap/dist/css/bootstrap.css.map similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.css.map rename to ui/static/bower_components/bootstrap/dist/css/bootstrap.css.map diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.min.css b/ui/static/bower_components/bootstrap/dist/css/bootstrap.min.css similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.min.css rename to ui/static/bower_components/bootstrap/dist/css/bootstrap.min.css diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.min.css.map b/ui/static/bower_components/bootstrap/dist/css/bootstrap.min.css.map similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/css/bootstrap.min.css.map rename to ui/static/bower_components/bootstrap/dist/css/bootstrap.min.css.map diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.eot b/ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.eot similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.eot rename to ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.eot diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.svg b/ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.svg similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.svg rename to ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.svg diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.ttf b/ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.ttf similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.ttf rename to ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.ttf diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff b/ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff rename to ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 b/ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 rename to ui/static/bower_components/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/js/bootstrap.js b/ui/static/bower_components/bootstrap/dist/js/bootstrap.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/js/bootstrap.js rename to ui/static/bower_components/bootstrap/dist/js/bootstrap.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/js/bootstrap.min.js b/ui/static/bower_components/bootstrap/dist/js/bootstrap.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/js/bootstrap.min.js rename to ui/static/bower_components/bootstrap/dist/js/bootstrap.min.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/dist/js/npm.js b/ui/static/bower_components/bootstrap/dist/js/npm.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/dist/js/npm.js rename to ui/static/bower_components/bootstrap/dist/js/npm.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.eot b/ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.eot similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.eot rename to ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.eot diff --git a/themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.svg b/ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.svg similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.svg rename to ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.svg diff --git a/themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.ttf b/ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.ttf similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.ttf rename to ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.ttf diff --git a/themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff b/ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff rename to ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff diff --git a/themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff2 b/ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff2 similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff2 rename to ui/static/bower_components/bootstrap/fonts/glyphicons-halflings-regular.woff2 diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/.jshintrc b/ui/static/bower_components/bootstrap/grunt/.jshintrc similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/.jshintrc rename to ui/static/bower_components/bootstrap/grunt/.jshintrc diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/bs-commonjs-generator.js b/ui/static/bower_components/bootstrap/grunt/bs-commonjs-generator.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/bs-commonjs-generator.js rename to ui/static/bower_components/bootstrap/grunt/bs-commonjs-generator.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/bs-glyphicons-data-generator.js b/ui/static/bower_components/bootstrap/grunt/bs-glyphicons-data-generator.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/bs-glyphicons-data-generator.js rename to ui/static/bower_components/bootstrap/grunt/bs-glyphicons-data-generator.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/bs-lessdoc-parser.js b/ui/static/bower_components/bootstrap/grunt/bs-lessdoc-parser.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/bs-lessdoc-parser.js rename to ui/static/bower_components/bootstrap/grunt/bs-lessdoc-parser.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/bs-raw-files-generator.js b/ui/static/bower_components/bootstrap/grunt/bs-raw-files-generator.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/bs-raw-files-generator.js rename to ui/static/bower_components/bootstrap/grunt/bs-raw-files-generator.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/change-version.js b/ui/static/bower_components/bootstrap/grunt/change-version.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/change-version.js rename to ui/static/bower_components/bootstrap/grunt/change-version.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/configBridge.json b/ui/static/bower_components/bootstrap/grunt/configBridge.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/configBridge.json rename to ui/static/bower_components/bootstrap/grunt/configBridge.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/npm-shrinkwrap.json b/ui/static/bower_components/bootstrap/grunt/npm-shrinkwrap.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/npm-shrinkwrap.json rename to ui/static/bower_components/bootstrap/grunt/npm-shrinkwrap.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/grunt/sauce_browsers.yml b/ui/static/bower_components/bootstrap/grunt/sauce_browsers.yml similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/grunt/sauce_browsers.yml rename to ui/static/bower_components/bootstrap/grunt/sauce_browsers.yml diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/.jscsrc b/ui/static/bower_components/bootstrap/js/.jscsrc similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/.jscsrc rename to ui/static/bower_components/bootstrap/js/.jscsrc diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/.jshintrc b/ui/static/bower_components/bootstrap/js/.jshintrc similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/.jshintrc rename to ui/static/bower_components/bootstrap/js/.jshintrc diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/affix.js b/ui/static/bower_components/bootstrap/js/affix.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/affix.js rename to ui/static/bower_components/bootstrap/js/affix.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/alert.js b/ui/static/bower_components/bootstrap/js/alert.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/alert.js rename to ui/static/bower_components/bootstrap/js/alert.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/button.js b/ui/static/bower_components/bootstrap/js/button.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/button.js rename to ui/static/bower_components/bootstrap/js/button.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/carousel.js b/ui/static/bower_components/bootstrap/js/carousel.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/carousel.js rename to ui/static/bower_components/bootstrap/js/carousel.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/collapse.js b/ui/static/bower_components/bootstrap/js/collapse.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/collapse.js rename to ui/static/bower_components/bootstrap/js/collapse.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/dropdown.js b/ui/static/bower_components/bootstrap/js/dropdown.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/dropdown.js rename to ui/static/bower_components/bootstrap/js/dropdown.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/modal.js b/ui/static/bower_components/bootstrap/js/modal.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/modal.js rename to ui/static/bower_components/bootstrap/js/modal.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/popover.js b/ui/static/bower_components/bootstrap/js/popover.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/popover.js rename to ui/static/bower_components/bootstrap/js/popover.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/scrollspy.js b/ui/static/bower_components/bootstrap/js/scrollspy.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/scrollspy.js rename to ui/static/bower_components/bootstrap/js/scrollspy.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/tab.js b/ui/static/bower_components/bootstrap/js/tab.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/tab.js rename to ui/static/bower_components/bootstrap/js/tab.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/tooltip.js b/ui/static/bower_components/bootstrap/js/tooltip.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/tooltip.js rename to ui/static/bower_components/bootstrap/js/tooltip.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/js/transition.js b/ui/static/bower_components/bootstrap/js/transition.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/js/transition.js rename to ui/static/bower_components/bootstrap/js/transition.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/.csscomb.json b/ui/static/bower_components/bootstrap/less/.csscomb.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/.csscomb.json rename to ui/static/bower_components/bootstrap/less/.csscomb.json diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/.csslintrc b/ui/static/bower_components/bootstrap/less/.csslintrc similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/.csslintrc rename to ui/static/bower_components/bootstrap/less/.csslintrc diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/alerts.less b/ui/static/bower_components/bootstrap/less/alerts.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/alerts.less rename to ui/static/bower_components/bootstrap/less/alerts.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/badges.less b/ui/static/bower_components/bootstrap/less/badges.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/badges.less rename to ui/static/bower_components/bootstrap/less/badges.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/bootstrap.less b/ui/static/bower_components/bootstrap/less/bootstrap.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/bootstrap.less rename to ui/static/bower_components/bootstrap/less/bootstrap.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/breadcrumbs.less b/ui/static/bower_components/bootstrap/less/breadcrumbs.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/breadcrumbs.less rename to ui/static/bower_components/bootstrap/less/breadcrumbs.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/button-groups.less b/ui/static/bower_components/bootstrap/less/button-groups.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/button-groups.less rename to ui/static/bower_components/bootstrap/less/button-groups.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/buttons.less b/ui/static/bower_components/bootstrap/less/buttons.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/buttons.less rename to ui/static/bower_components/bootstrap/less/buttons.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/carousel.less b/ui/static/bower_components/bootstrap/less/carousel.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/carousel.less rename to ui/static/bower_components/bootstrap/less/carousel.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/close.less b/ui/static/bower_components/bootstrap/less/close.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/close.less rename to ui/static/bower_components/bootstrap/less/close.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/code.less b/ui/static/bower_components/bootstrap/less/code.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/code.less rename to ui/static/bower_components/bootstrap/less/code.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/component-animations.less b/ui/static/bower_components/bootstrap/less/component-animations.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/component-animations.less rename to ui/static/bower_components/bootstrap/less/component-animations.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/dropdowns.less b/ui/static/bower_components/bootstrap/less/dropdowns.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/dropdowns.less rename to ui/static/bower_components/bootstrap/less/dropdowns.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/forms.less b/ui/static/bower_components/bootstrap/less/forms.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/forms.less rename to ui/static/bower_components/bootstrap/less/forms.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/glyphicons.less b/ui/static/bower_components/bootstrap/less/glyphicons.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/glyphicons.less rename to ui/static/bower_components/bootstrap/less/glyphicons.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/grid.less b/ui/static/bower_components/bootstrap/less/grid.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/grid.less rename to ui/static/bower_components/bootstrap/less/grid.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/input-groups.less b/ui/static/bower_components/bootstrap/less/input-groups.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/input-groups.less rename to ui/static/bower_components/bootstrap/less/input-groups.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/jumbotron.less b/ui/static/bower_components/bootstrap/less/jumbotron.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/jumbotron.less rename to ui/static/bower_components/bootstrap/less/jumbotron.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/labels.less b/ui/static/bower_components/bootstrap/less/labels.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/labels.less rename to ui/static/bower_components/bootstrap/less/labels.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/list-group.less b/ui/static/bower_components/bootstrap/less/list-group.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/list-group.less rename to ui/static/bower_components/bootstrap/less/list-group.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/media.less b/ui/static/bower_components/bootstrap/less/media.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/media.less rename to ui/static/bower_components/bootstrap/less/media.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins.less b/ui/static/bower_components/bootstrap/less/mixins.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins.less rename to ui/static/bower_components/bootstrap/less/mixins.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/alerts.less b/ui/static/bower_components/bootstrap/less/mixins/alerts.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/alerts.less rename to ui/static/bower_components/bootstrap/less/mixins/alerts.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/background-variant.less b/ui/static/bower_components/bootstrap/less/mixins/background-variant.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/background-variant.less rename to ui/static/bower_components/bootstrap/less/mixins/background-variant.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/border-radius.less b/ui/static/bower_components/bootstrap/less/mixins/border-radius.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/border-radius.less rename to ui/static/bower_components/bootstrap/less/mixins/border-radius.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/buttons.less b/ui/static/bower_components/bootstrap/less/mixins/buttons.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/buttons.less rename to ui/static/bower_components/bootstrap/less/mixins/buttons.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/center-block.less b/ui/static/bower_components/bootstrap/less/mixins/center-block.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/center-block.less rename to ui/static/bower_components/bootstrap/less/mixins/center-block.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/clearfix.less b/ui/static/bower_components/bootstrap/less/mixins/clearfix.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/clearfix.less rename to ui/static/bower_components/bootstrap/less/mixins/clearfix.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/forms.less b/ui/static/bower_components/bootstrap/less/mixins/forms.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/forms.less rename to ui/static/bower_components/bootstrap/less/mixins/forms.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/gradients.less b/ui/static/bower_components/bootstrap/less/mixins/gradients.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/gradients.less rename to ui/static/bower_components/bootstrap/less/mixins/gradients.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/grid-framework.less b/ui/static/bower_components/bootstrap/less/mixins/grid-framework.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/grid-framework.less rename to ui/static/bower_components/bootstrap/less/mixins/grid-framework.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/grid.less b/ui/static/bower_components/bootstrap/less/mixins/grid.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/grid.less rename to ui/static/bower_components/bootstrap/less/mixins/grid.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/hide-text.less b/ui/static/bower_components/bootstrap/less/mixins/hide-text.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/hide-text.less rename to ui/static/bower_components/bootstrap/less/mixins/hide-text.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/image.less b/ui/static/bower_components/bootstrap/less/mixins/image.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/image.less rename to ui/static/bower_components/bootstrap/less/mixins/image.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/labels.less b/ui/static/bower_components/bootstrap/less/mixins/labels.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/labels.less rename to ui/static/bower_components/bootstrap/less/mixins/labels.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/list-group.less b/ui/static/bower_components/bootstrap/less/mixins/list-group.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/list-group.less rename to ui/static/bower_components/bootstrap/less/mixins/list-group.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/nav-divider.less b/ui/static/bower_components/bootstrap/less/mixins/nav-divider.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/nav-divider.less rename to ui/static/bower_components/bootstrap/less/mixins/nav-divider.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/nav-vertical-align.less b/ui/static/bower_components/bootstrap/less/mixins/nav-vertical-align.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/nav-vertical-align.less rename to ui/static/bower_components/bootstrap/less/mixins/nav-vertical-align.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/opacity.less b/ui/static/bower_components/bootstrap/less/mixins/opacity.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/opacity.less rename to ui/static/bower_components/bootstrap/less/mixins/opacity.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/pagination.less b/ui/static/bower_components/bootstrap/less/mixins/pagination.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/pagination.less rename to ui/static/bower_components/bootstrap/less/mixins/pagination.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/panels.less b/ui/static/bower_components/bootstrap/less/mixins/panels.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/panels.less rename to ui/static/bower_components/bootstrap/less/mixins/panels.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/progress-bar.less b/ui/static/bower_components/bootstrap/less/mixins/progress-bar.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/progress-bar.less rename to ui/static/bower_components/bootstrap/less/mixins/progress-bar.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/reset-filter.less b/ui/static/bower_components/bootstrap/less/mixins/reset-filter.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/reset-filter.less rename to ui/static/bower_components/bootstrap/less/mixins/reset-filter.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/reset-text.less b/ui/static/bower_components/bootstrap/less/mixins/reset-text.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/reset-text.less rename to ui/static/bower_components/bootstrap/less/mixins/reset-text.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/resize.less b/ui/static/bower_components/bootstrap/less/mixins/resize.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/resize.less rename to ui/static/bower_components/bootstrap/less/mixins/resize.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/responsive-visibility.less b/ui/static/bower_components/bootstrap/less/mixins/responsive-visibility.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/responsive-visibility.less rename to ui/static/bower_components/bootstrap/less/mixins/responsive-visibility.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/size.less b/ui/static/bower_components/bootstrap/less/mixins/size.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/size.less rename to ui/static/bower_components/bootstrap/less/mixins/size.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/tab-focus.less b/ui/static/bower_components/bootstrap/less/mixins/tab-focus.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/tab-focus.less rename to ui/static/bower_components/bootstrap/less/mixins/tab-focus.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/table-row.less b/ui/static/bower_components/bootstrap/less/mixins/table-row.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/table-row.less rename to ui/static/bower_components/bootstrap/less/mixins/table-row.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/text-emphasis.less b/ui/static/bower_components/bootstrap/less/mixins/text-emphasis.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/text-emphasis.less rename to ui/static/bower_components/bootstrap/less/mixins/text-emphasis.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/text-overflow.less b/ui/static/bower_components/bootstrap/less/mixins/text-overflow.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/text-overflow.less rename to ui/static/bower_components/bootstrap/less/mixins/text-overflow.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/mixins/vendor-prefixes.less b/ui/static/bower_components/bootstrap/less/mixins/vendor-prefixes.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/mixins/vendor-prefixes.less rename to ui/static/bower_components/bootstrap/less/mixins/vendor-prefixes.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/modals.less b/ui/static/bower_components/bootstrap/less/modals.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/modals.less rename to ui/static/bower_components/bootstrap/less/modals.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/navbar.less b/ui/static/bower_components/bootstrap/less/navbar.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/navbar.less rename to ui/static/bower_components/bootstrap/less/navbar.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/navs.less b/ui/static/bower_components/bootstrap/less/navs.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/navs.less rename to ui/static/bower_components/bootstrap/less/navs.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/normalize.less b/ui/static/bower_components/bootstrap/less/normalize.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/normalize.less rename to ui/static/bower_components/bootstrap/less/normalize.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/pager.less b/ui/static/bower_components/bootstrap/less/pager.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/pager.less rename to ui/static/bower_components/bootstrap/less/pager.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/pagination.less b/ui/static/bower_components/bootstrap/less/pagination.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/pagination.less rename to ui/static/bower_components/bootstrap/less/pagination.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/panels.less b/ui/static/bower_components/bootstrap/less/panels.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/panels.less rename to ui/static/bower_components/bootstrap/less/panels.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/popovers.less b/ui/static/bower_components/bootstrap/less/popovers.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/popovers.less rename to ui/static/bower_components/bootstrap/less/popovers.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/print.less b/ui/static/bower_components/bootstrap/less/print.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/print.less rename to ui/static/bower_components/bootstrap/less/print.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/progress-bars.less b/ui/static/bower_components/bootstrap/less/progress-bars.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/progress-bars.less rename to ui/static/bower_components/bootstrap/less/progress-bars.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/responsive-embed.less b/ui/static/bower_components/bootstrap/less/responsive-embed.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/responsive-embed.less rename to ui/static/bower_components/bootstrap/less/responsive-embed.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/responsive-utilities.less b/ui/static/bower_components/bootstrap/less/responsive-utilities.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/responsive-utilities.less rename to ui/static/bower_components/bootstrap/less/responsive-utilities.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/scaffolding.less b/ui/static/bower_components/bootstrap/less/scaffolding.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/scaffolding.less rename to ui/static/bower_components/bootstrap/less/scaffolding.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/tables.less b/ui/static/bower_components/bootstrap/less/tables.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/tables.less rename to ui/static/bower_components/bootstrap/less/tables.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/theme.less b/ui/static/bower_components/bootstrap/less/theme.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/theme.less rename to ui/static/bower_components/bootstrap/less/theme.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/thumbnails.less b/ui/static/bower_components/bootstrap/less/thumbnails.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/thumbnails.less rename to ui/static/bower_components/bootstrap/less/thumbnails.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/tooltip.less b/ui/static/bower_components/bootstrap/less/tooltip.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/tooltip.less rename to ui/static/bower_components/bootstrap/less/tooltip.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/type.less b/ui/static/bower_components/bootstrap/less/type.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/type.less rename to ui/static/bower_components/bootstrap/less/type.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/utilities.less b/ui/static/bower_components/bootstrap/less/utilities.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/utilities.less rename to ui/static/bower_components/bootstrap/less/utilities.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/variables.less b/ui/static/bower_components/bootstrap/less/variables.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/variables.less rename to ui/static/bower_components/bootstrap/less/variables.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/less/wells.less b/ui/static/bower_components/bootstrap/less/wells.less similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/less/wells.less rename to ui/static/bower_components/bootstrap/less/wells.less diff --git a/themes/bootstrap/public/bower_components/bootstrap/nuget/MyGet.ps1 b/ui/static/bower_components/bootstrap/nuget/MyGet.ps1 similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/nuget/MyGet.ps1 rename to ui/static/bower_components/bootstrap/nuget/MyGet.ps1 diff --git a/themes/bootstrap/public/bower_components/bootstrap/nuget/bootstrap.less.nuspec b/ui/static/bower_components/bootstrap/nuget/bootstrap.less.nuspec similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/nuget/bootstrap.less.nuspec rename to ui/static/bower_components/bootstrap/nuget/bootstrap.less.nuspec diff --git a/themes/bootstrap/public/bower_components/bootstrap/nuget/bootstrap.nuspec b/ui/static/bower_components/bootstrap/nuget/bootstrap.nuspec similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/nuget/bootstrap.nuspec rename to ui/static/bower_components/bootstrap/nuget/bootstrap.nuspec diff --git a/themes/bootstrap/public/bower_components/bootstrap/package.js b/ui/static/bower_components/bootstrap/package.js similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/package.js rename to ui/static/bower_components/bootstrap/package.js diff --git a/themes/bootstrap/public/bower_components/bootstrap/package.json b/ui/static/bower_components/bootstrap/package.json similarity index 100% rename from themes/bootstrap/public/bower_components/bootstrap/package.json rename to ui/static/bower_components/bootstrap/package.json diff --git a/themes/bootstrap/public/bower_components/clipboard/.bower.json b/ui/static/bower_components/clipboard/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/.bower.json rename to ui/static/bower_components/clipboard/.bower.json diff --git a/themes/bootstrap/public/bower_components/clipboard/bower.json b/ui/static/bower_components/clipboard/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/bower.json rename to ui/static/bower_components/clipboard/bower.json diff --git a/themes/bootstrap/public/bower_components/clipboard/contributing.md b/ui/static/bower_components/clipboard/contributing.md similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/contributing.md rename to ui/static/bower_components/clipboard/contributing.md diff --git a/themes/bootstrap/public/bower_components/clipboard/dist/clipboard.js b/ui/static/bower_components/clipboard/dist/clipboard.js similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/dist/clipboard.js rename to ui/static/bower_components/clipboard/dist/clipboard.js diff --git a/themes/bootstrap/public/bower_components/clipboard/dist/clipboard.min.js b/ui/static/bower_components/clipboard/dist/clipboard.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/dist/clipboard.min.js rename to ui/static/bower_components/clipboard/dist/clipboard.min.js diff --git a/themes/bootstrap/public/bower_components/clipboard/package.js b/ui/static/bower_components/clipboard/package.js similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/package.js rename to ui/static/bower_components/clipboard/package.js diff --git a/themes/bootstrap/public/bower_components/clipboard/package.json b/ui/static/bower_components/clipboard/package.json similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/package.json rename to ui/static/bower_components/clipboard/package.json diff --git a/themes/bootstrap/public/bower_components/clipboard/readme.md b/ui/static/bower_components/clipboard/readme.md similarity index 100% rename from themes/bootstrap/public/bower_components/clipboard/readme.md rename to ui/static/bower_components/clipboard/readme.md diff --git a/themes/bootstrap/public/bower_components/jquery-color/.bower.json b/ui/static/bower_components/jquery-color/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/.bower.json rename to ui/static/bower_components/jquery-color/.bower.json diff --git a/themes/bootstrap/public/bower_components/jquery-color/.gitignore b/ui/static/bower_components/jquery-color/.gitignore similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/.gitignore rename to ui/static/bower_components/jquery-color/.gitignore diff --git a/themes/bootstrap/public/bower_components/jquery-color/.gitmodules b/ui/static/bower_components/jquery-color/.gitmodules similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/.gitmodules rename to ui/static/bower_components/jquery-color/.gitmodules diff --git a/themes/bootstrap/public/bower_components/jquery-color/.jshintrc b/ui/static/bower_components/jquery-color/.jshintrc similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/.jshintrc rename to ui/static/bower_components/jquery-color/.jshintrc diff --git a/themes/bootstrap/public/bower_components/jquery-color/AUTHORS.TXT b/ui/static/bower_components/jquery-color/AUTHORS.TXT similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/AUTHORS.TXT rename to ui/static/bower_components/jquery-color/AUTHORS.TXT diff --git a/themes/bootstrap/public/bower_components/jquery-color/MIT-LICENSE.txt b/ui/static/bower_components/jquery-color/MIT-LICENSE.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/MIT-LICENSE.txt rename to ui/static/bower_components/jquery-color/MIT-LICENSE.txt diff --git a/themes/bootstrap/public/bower_components/jquery-color/README.md b/ui/static/bower_components/jquery-color/README.md similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/README.md rename to ui/static/bower_components/jquery-color/README.md diff --git a/themes/bootstrap/public/bower_components/jquery-color/color.jquery.json b/ui/static/bower_components/jquery-color/color.jquery.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/color.jquery.json rename to ui/static/bower_components/jquery-color/color.jquery.json diff --git a/themes/bootstrap/public/bower_components/jquery-color/grunt.js b/ui/static/bower_components/jquery-color/grunt.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/grunt.js rename to ui/static/bower_components/jquery-color/grunt.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/jquery.color.js b/ui/static/bower_components/jquery-color/jquery.color.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/jquery.color.js rename to ui/static/bower_components/jquery-color/jquery.color.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/jquery.color.svg-names.js b/ui/static/bower_components/jquery-color/jquery.color.svg-names.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/jquery.color.svg-names.js rename to ui/static/bower_components/jquery-color/jquery.color.svg-names.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/package.json b/ui/static/bower_components/jquery-color/package.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/package.json rename to ui/static/bower_components/jquery-color/package.json diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/data/swarminject.js b/ui/static/bower_components/jquery-color/test/data/swarminject.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/data/swarminject.js rename to ui/static/bower_components/jquery-color/test/data/swarminject.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/data/testinit.js b/ui/static/bower_components/jquery-color/test/data/testinit.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/data/testinit.js rename to ui/static/bower_components/jquery-color/test/data/testinit.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/index.html b/ui/static/bower_components/jquery-color/test/index.html similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/index.html rename to ui/static/bower_components/jquery-color/test/index.html diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.1.js b/ui/static/bower_components/jquery-color/test/jquery-1.5.1.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.1.js rename to ui/static/bower_components/jquery-color/test/jquery-1.5.1.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.2.js b/ui/static/bower_components/jquery-color/test/jquery-1.5.2.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.2.js rename to ui/static/bower_components/jquery-color/test/jquery-1.5.2.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.js b/ui/static/bower_components/jquery-color/test/jquery-1.5.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.5.js rename to ui/static/bower_components/jquery-color/test/jquery-1.5.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.1.js b/ui/static/bower_components/jquery-color/test/jquery-1.6.1.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.1.js rename to ui/static/bower_components/jquery-color/test/jquery-1.6.1.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.2.js b/ui/static/bower_components/jquery-color/test/jquery-1.6.2.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.2.js rename to ui/static/bower_components/jquery-color/test/jquery-1.6.2.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.3.js b/ui/static/bower_components/jquery-color/test/jquery-1.6.3.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.3.js rename to ui/static/bower_components/jquery-color/test/jquery-1.6.3.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.4.js b/ui/static/bower_components/jquery-color/test/jquery-1.6.4.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.4.js rename to ui/static/bower_components/jquery-color/test/jquery-1.6.4.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.js b/ui/static/bower_components/jquery-color/test/jquery-1.6.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.6.js rename to ui/static/bower_components/jquery-color/test/jquery-1.6.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.1.js b/ui/static/bower_components/jquery-color/test/jquery-1.7.1.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.1.js rename to ui/static/bower_components/jquery-color/test/jquery-1.7.1.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.2.js b/ui/static/bower_components/jquery-color/test/jquery-1.7.2.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.2.js rename to ui/static/bower_components/jquery-color/test/jquery-1.7.2.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.js b/ui/static/bower_components/jquery-color/test/jquery-1.7.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery-1.7.js rename to ui/static/bower_components/jquery-color/test/jquery-1.7.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/jquery.js b/ui/static/bower_components/jquery-color/test/jquery.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/jquery.js rename to ui/static/bower_components/jquery-color/test/jquery.js diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/test.html b/ui/static/bower_components/jquery-color/test/test.html similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/test.html rename to ui/static/bower_components/jquery-color/test/test.html diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/unit/.jshintrc b/ui/static/bower_components/jquery-color/test/unit/.jshintrc similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/unit/.jshintrc rename to ui/static/bower_components/jquery-color/test/unit/.jshintrc diff --git a/themes/bootstrap/public/bower_components/jquery-color/test/unit/color.js b/ui/static/bower_components/jquery-color/test/unit/color.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-color/test/unit/color.js rename to ui/static/bower_components/jquery-color/test/unit/color.js diff --git a/themes/bootstrap/public/bower_components/jquery-load-template/.bower.json b/ui/static/bower_components/jquery-load-template/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-load-template/.bower.json rename to ui/static/bower_components/jquery-load-template/.bower.json diff --git a/themes/bootstrap/public/bower_components/jquery-load-template/bower.json b/ui/static/bower_components/jquery-load-template/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-load-template/bower.json rename to ui/static/bower_components/jquery-load-template/bower.json diff --git a/themes/bootstrap/public/bower_components/jquery-load-template/dist/jquery.loadTemplate.js b/ui/static/bower_components/jquery-load-template/dist/jquery.loadTemplate.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-load-template/dist/jquery.loadTemplate.js rename to ui/static/bower_components/jquery-load-template/dist/jquery.loadTemplate.js diff --git a/themes/bootstrap/public/bower_components/jquery-load-template/dist/jquery.loadTemplate.min.js b/ui/static/bower_components/jquery-load-template/dist/jquery.loadTemplate.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-load-template/dist/jquery.loadTemplate.min.js rename to ui/static/bower_components/jquery-load-template/dist/jquery.loadTemplate.min.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/.bower.json b/ui/static/bower_components/jquery-sparkline/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/.bower.json rename to ui/static/bower_components/jquery-sparkline/.bower.json diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/Changelog.txt b/ui/static/bower_components/jquery-sparkline/Changelog.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/Changelog.txt rename to ui/static/bower_components/jquery-sparkline/Changelog.txt diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/Makefile b/ui/static/bower_components/jquery-sparkline/Makefile similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/Makefile rename to ui/static/bower_components/jquery-sparkline/Makefile diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/README.md b/ui/static/bower_components/jquery-sparkline/README.md similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/README.md rename to ui/static/bower_components/jquery-sparkline/README.md diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/bower.json b/ui/static/bower_components/jquery-sparkline/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/bower.json rename to ui/static/bower_components/jquery-sparkline/bower.json diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/dist/jquery.sparkline.js b/ui/static/bower_components/jquery-sparkline/dist/jquery.sparkline.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/dist/jquery.sparkline.js rename to ui/static/bower_components/jquery-sparkline/dist/jquery.sparkline.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/dist/jquery.sparkline.min.js b/ui/static/bower_components/jquery-sparkline/dist/jquery.sparkline.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/dist/jquery.sparkline.min.js rename to ui/static/bower_components/jquery-sparkline/dist/jquery.sparkline.min.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/minheader.txt b/ui/static/bower_components/jquery-sparkline/minheader.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/minheader.txt rename to ui/static/bower_components/jquery-sparkline/minheader.txt diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/sparkline.jquery.json b/ui/static/bower_components/jquery-sparkline/sparkline.jquery.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/sparkline.jquery.json rename to ui/static/bower_components/jquery-sparkline/sparkline.jquery.json diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/base.js b/ui/static/bower_components/jquery-sparkline/src/base.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/base.js rename to ui/static/bower_components/jquery-sparkline/src/base.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-bar.js b/ui/static/bower_components/jquery-sparkline/src/chart-bar.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-bar.js rename to ui/static/bower_components/jquery-sparkline/src/chart-bar.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-box.js b/ui/static/bower_components/jquery-sparkline/src/chart-box.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-box.js rename to ui/static/bower_components/jquery-sparkline/src/chart-box.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-bullet.js b/ui/static/bower_components/jquery-sparkline/src/chart-bullet.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-bullet.js rename to ui/static/bower_components/jquery-sparkline/src/chart-bullet.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-discrete.js b/ui/static/bower_components/jquery-sparkline/src/chart-discrete.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-discrete.js rename to ui/static/bower_components/jquery-sparkline/src/chart-discrete.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-line.js b/ui/static/bower_components/jquery-sparkline/src/chart-line.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-line.js rename to ui/static/bower_components/jquery-sparkline/src/chart-line.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-pie.js b/ui/static/bower_components/jquery-sparkline/src/chart-pie.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-pie.js rename to ui/static/bower_components/jquery-sparkline/src/chart-pie.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-tristate.js b/ui/static/bower_components/jquery-sparkline/src/chart-tristate.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/chart-tristate.js rename to ui/static/bower_components/jquery-sparkline/src/chart-tristate.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/defaults.js b/ui/static/bower_components/jquery-sparkline/src/defaults.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/defaults.js rename to ui/static/bower_components/jquery-sparkline/src/defaults.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/footer.js b/ui/static/bower_components/jquery-sparkline/src/footer.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/footer.js rename to ui/static/bower_components/jquery-sparkline/src/footer.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/header.js b/ui/static/bower_components/jquery-sparkline/src/header.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/header.js rename to ui/static/bower_components/jquery-sparkline/src/header.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/interact.js b/ui/static/bower_components/jquery-sparkline/src/interact.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/interact.js rename to ui/static/bower_components/jquery-sparkline/src/interact.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/rangemap.js b/ui/static/bower_components/jquery-sparkline/src/rangemap.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/rangemap.js rename to ui/static/bower_components/jquery-sparkline/src/rangemap.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/simpledraw.js b/ui/static/bower_components/jquery-sparkline/src/simpledraw.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/simpledraw.js rename to ui/static/bower_components/jquery-sparkline/src/simpledraw.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/utils.js b/ui/static/bower_components/jquery-sparkline/src/utils.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/utils.js rename to ui/static/bower_components/jquery-sparkline/src/utils.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-base.js b/ui/static/bower_components/jquery-sparkline/src/vcanvas-base.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-base.js rename to ui/static/bower_components/jquery-sparkline/src/vcanvas-base.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-canvas.js b/ui/static/bower_components/jquery-sparkline/src/vcanvas-canvas.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-canvas.js rename to ui/static/bower_components/jquery-sparkline/src/vcanvas-canvas.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-vml.js b/ui/static/bower_components/jquery-sparkline/src/vcanvas-vml.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/src/vcanvas-vml.js rename to ui/static/bower_components/jquery-sparkline/src/vcanvas-vml.js diff --git a/themes/bootstrap/public/bower_components/jquery-sparkline/version.txt b/ui/static/bower_components/jquery-sparkline/version.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery-sparkline/version.txt rename to ui/static/bower_components/jquery-sparkline/version.txt diff --git a/themes/bootstrap/public/bower_components/jquery/.bower.json b/ui/static/bower_components/jquery/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/.bower.json rename to ui/static/bower_components/jquery/.bower.json diff --git a/themes/bootstrap/public/bower_components/jquery/AUTHORS.txt b/ui/static/bower_components/jquery/AUTHORS.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/AUTHORS.txt rename to ui/static/bower_components/jquery/AUTHORS.txt diff --git a/themes/bootstrap/public/bower_components/jquery/LICENSE.txt b/ui/static/bower_components/jquery/LICENSE.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/LICENSE.txt rename to ui/static/bower_components/jquery/LICENSE.txt diff --git a/themes/bootstrap/public/bower_components/jquery/README.md b/ui/static/bower_components/jquery/README.md similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/README.md rename to ui/static/bower_components/jquery/README.md diff --git a/themes/bootstrap/public/bower_components/jquery/bower.json b/ui/static/bower_components/jquery/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/bower.json rename to ui/static/bower_components/jquery/bower.json diff --git a/themes/bootstrap/public/bower_components/jquery/dist/jquery.js b/ui/static/bower_components/jquery/dist/jquery.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/dist/jquery.js rename to ui/static/bower_components/jquery/dist/jquery.js diff --git a/themes/bootstrap/public/bower_components/jquery/dist/jquery.min.js b/ui/static/bower_components/jquery/dist/jquery.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/dist/jquery.min.js rename to ui/static/bower_components/jquery/dist/jquery.min.js diff --git a/themes/bootstrap/public/bower_components/jquery/dist/jquery.min.map b/ui/static/bower_components/jquery/dist/jquery.min.map similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/dist/jquery.min.map rename to ui/static/bower_components/jquery/dist/jquery.min.map diff --git a/themes/bootstrap/public/bower_components/jquery/external/sizzle/LICENSE.txt b/ui/static/bower_components/jquery/external/sizzle/LICENSE.txt similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/external/sizzle/LICENSE.txt rename to ui/static/bower_components/jquery/external/sizzle/LICENSE.txt diff --git a/themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.js b/ui/static/bower_components/jquery/external/sizzle/dist/sizzle.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.js rename to ui/static/bower_components/jquery/external/sizzle/dist/sizzle.js diff --git a/themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.min.js b/ui/static/bower_components/jquery/external/sizzle/dist/sizzle.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.min.js rename to ui/static/bower_components/jquery/external/sizzle/dist/sizzle.min.js diff --git a/themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.min.map b/ui/static/bower_components/jquery/external/sizzle/dist/sizzle.min.map similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/external/sizzle/dist/sizzle.min.map rename to ui/static/bower_components/jquery/external/sizzle/dist/sizzle.min.map diff --git a/themes/bootstrap/public/bower_components/jquery/src/.jshintrc b/ui/static/bower_components/jquery/src/.jshintrc similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/.jshintrc rename to ui/static/bower_components/jquery/src/.jshintrc diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax.js b/ui/static/bower_components/jquery/src/ajax.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax.js rename to ui/static/bower_components/jquery/src/ajax.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/jsonp.js b/ui/static/bower_components/jquery/src/ajax/jsonp.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/jsonp.js rename to ui/static/bower_components/jquery/src/ajax/jsonp.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/load.js b/ui/static/bower_components/jquery/src/ajax/load.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/load.js rename to ui/static/bower_components/jquery/src/ajax/load.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/parseJSON.js b/ui/static/bower_components/jquery/src/ajax/parseJSON.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/parseJSON.js rename to ui/static/bower_components/jquery/src/ajax/parseJSON.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/parseXML.js b/ui/static/bower_components/jquery/src/ajax/parseXML.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/parseXML.js rename to ui/static/bower_components/jquery/src/ajax/parseXML.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/script.js b/ui/static/bower_components/jquery/src/ajax/script.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/script.js rename to ui/static/bower_components/jquery/src/ajax/script.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/var/location.js b/ui/static/bower_components/jquery/src/ajax/var/location.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/var/location.js rename to ui/static/bower_components/jquery/src/ajax/var/location.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/var/nonce.js b/ui/static/bower_components/jquery/src/ajax/var/nonce.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/var/nonce.js rename to ui/static/bower_components/jquery/src/ajax/var/nonce.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/var/rquery.js b/ui/static/bower_components/jquery/src/ajax/var/rquery.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/var/rquery.js rename to ui/static/bower_components/jquery/src/ajax/var/rquery.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/ajax/xhr.js b/ui/static/bower_components/jquery/src/ajax/xhr.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/ajax/xhr.js rename to ui/static/bower_components/jquery/src/ajax/xhr.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes.js b/ui/static/bower_components/jquery/src/attributes.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes.js rename to ui/static/bower_components/jquery/src/attributes.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes/attr.js b/ui/static/bower_components/jquery/src/attributes/attr.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes/attr.js rename to ui/static/bower_components/jquery/src/attributes/attr.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes/classes.js b/ui/static/bower_components/jquery/src/attributes/classes.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes/classes.js rename to ui/static/bower_components/jquery/src/attributes/classes.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes/prop.js b/ui/static/bower_components/jquery/src/attributes/prop.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes/prop.js rename to ui/static/bower_components/jquery/src/attributes/prop.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes/support.js b/ui/static/bower_components/jquery/src/attributes/support.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes/support.js rename to ui/static/bower_components/jquery/src/attributes/support.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/attributes/val.js b/ui/static/bower_components/jquery/src/attributes/val.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/attributes/val.js rename to ui/static/bower_components/jquery/src/attributes/val.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/callbacks.js b/ui/static/bower_components/jquery/src/callbacks.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/callbacks.js rename to ui/static/bower_components/jquery/src/callbacks.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core.js b/ui/static/bower_components/jquery/src/core.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core.js rename to ui/static/bower_components/jquery/src/core.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core/access.js b/ui/static/bower_components/jquery/src/core/access.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core/access.js rename to ui/static/bower_components/jquery/src/core/access.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core/init.js b/ui/static/bower_components/jquery/src/core/init.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core/init.js rename to ui/static/bower_components/jquery/src/core/init.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core/parseHTML.js b/ui/static/bower_components/jquery/src/core/parseHTML.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core/parseHTML.js rename to ui/static/bower_components/jquery/src/core/parseHTML.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core/ready.js b/ui/static/bower_components/jquery/src/core/ready.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core/ready.js rename to ui/static/bower_components/jquery/src/core/ready.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/core/var/rsingleTag.js b/ui/static/bower_components/jquery/src/core/var/rsingleTag.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/core/var/rsingleTag.js rename to ui/static/bower_components/jquery/src/core/var/rsingleTag.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css.js b/ui/static/bower_components/jquery/src/css.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css.js rename to ui/static/bower_components/jquery/src/css.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/addGetHookIf.js b/ui/static/bower_components/jquery/src/css/addGetHookIf.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/addGetHookIf.js rename to ui/static/bower_components/jquery/src/css/addGetHookIf.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/adjustCSS.js b/ui/static/bower_components/jquery/src/css/adjustCSS.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/adjustCSS.js rename to ui/static/bower_components/jquery/src/css/adjustCSS.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/curCSS.js b/ui/static/bower_components/jquery/src/css/curCSS.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/curCSS.js rename to ui/static/bower_components/jquery/src/css/curCSS.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/defaultDisplay.js b/ui/static/bower_components/jquery/src/css/defaultDisplay.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/defaultDisplay.js rename to ui/static/bower_components/jquery/src/css/defaultDisplay.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/hiddenVisibleSelectors.js b/ui/static/bower_components/jquery/src/css/hiddenVisibleSelectors.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/hiddenVisibleSelectors.js rename to ui/static/bower_components/jquery/src/css/hiddenVisibleSelectors.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/showHide.js b/ui/static/bower_components/jquery/src/css/showHide.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/showHide.js rename to ui/static/bower_components/jquery/src/css/showHide.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/support.js b/ui/static/bower_components/jquery/src/css/support.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/support.js rename to ui/static/bower_components/jquery/src/css/support.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/cssExpand.js b/ui/static/bower_components/jquery/src/css/var/cssExpand.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/cssExpand.js rename to ui/static/bower_components/jquery/src/css/var/cssExpand.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/getStyles.js b/ui/static/bower_components/jquery/src/css/var/getStyles.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/getStyles.js rename to ui/static/bower_components/jquery/src/css/var/getStyles.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/isHidden.js b/ui/static/bower_components/jquery/src/css/var/isHidden.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/isHidden.js rename to ui/static/bower_components/jquery/src/css/var/isHidden.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/rmargin.js b/ui/static/bower_components/jquery/src/css/var/rmargin.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/rmargin.js rename to ui/static/bower_components/jquery/src/css/var/rmargin.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/rnumnonpx.js b/ui/static/bower_components/jquery/src/css/var/rnumnonpx.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/rnumnonpx.js rename to ui/static/bower_components/jquery/src/css/var/rnumnonpx.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/css/var/swap.js b/ui/static/bower_components/jquery/src/css/var/swap.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/css/var/swap.js rename to ui/static/bower_components/jquery/src/css/var/swap.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/data.js b/ui/static/bower_components/jquery/src/data.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/data.js rename to ui/static/bower_components/jquery/src/data.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/data/Data.js b/ui/static/bower_components/jquery/src/data/Data.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/data/Data.js rename to ui/static/bower_components/jquery/src/data/Data.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/data/var/acceptData.js b/ui/static/bower_components/jquery/src/data/var/acceptData.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/data/var/acceptData.js rename to ui/static/bower_components/jquery/src/data/var/acceptData.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/data/var/dataPriv.js b/ui/static/bower_components/jquery/src/data/var/dataPriv.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/data/var/dataPriv.js rename to ui/static/bower_components/jquery/src/data/var/dataPriv.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/data/var/dataUser.js b/ui/static/bower_components/jquery/src/data/var/dataUser.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/data/var/dataUser.js rename to ui/static/bower_components/jquery/src/data/var/dataUser.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/deferred.js b/ui/static/bower_components/jquery/src/deferred.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/deferred.js rename to ui/static/bower_components/jquery/src/deferred.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/deprecated.js b/ui/static/bower_components/jquery/src/deprecated.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/deprecated.js rename to ui/static/bower_components/jquery/src/deprecated.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/dimensions.js b/ui/static/bower_components/jquery/src/dimensions.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/dimensions.js rename to ui/static/bower_components/jquery/src/dimensions.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/effects.js b/ui/static/bower_components/jquery/src/effects.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/effects.js rename to ui/static/bower_components/jquery/src/effects.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/effects/Tween.js b/ui/static/bower_components/jquery/src/effects/Tween.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/effects/Tween.js rename to ui/static/bower_components/jquery/src/effects/Tween.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/effects/animatedSelector.js b/ui/static/bower_components/jquery/src/effects/animatedSelector.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/effects/animatedSelector.js rename to ui/static/bower_components/jquery/src/effects/animatedSelector.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event.js b/ui/static/bower_components/jquery/src/event.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event.js rename to ui/static/bower_components/jquery/src/event.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event/ajax.js b/ui/static/bower_components/jquery/src/event/ajax.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event/ajax.js rename to ui/static/bower_components/jquery/src/event/ajax.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event/alias.js b/ui/static/bower_components/jquery/src/event/alias.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event/alias.js rename to ui/static/bower_components/jquery/src/event/alias.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event/focusin.js b/ui/static/bower_components/jquery/src/event/focusin.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event/focusin.js rename to ui/static/bower_components/jquery/src/event/focusin.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event/support.js b/ui/static/bower_components/jquery/src/event/support.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event/support.js rename to ui/static/bower_components/jquery/src/event/support.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/event/trigger.js b/ui/static/bower_components/jquery/src/event/trigger.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/event/trigger.js rename to ui/static/bower_components/jquery/src/event/trigger.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/exports/amd.js b/ui/static/bower_components/jquery/src/exports/amd.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/exports/amd.js rename to ui/static/bower_components/jquery/src/exports/amd.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/exports/global.js b/ui/static/bower_components/jquery/src/exports/global.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/exports/global.js rename to ui/static/bower_components/jquery/src/exports/global.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/intro.js b/ui/static/bower_components/jquery/src/intro.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/intro.js rename to ui/static/bower_components/jquery/src/intro.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/jquery.js b/ui/static/bower_components/jquery/src/jquery.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/jquery.js rename to ui/static/bower_components/jquery/src/jquery.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation.js b/ui/static/bower_components/jquery/src/manipulation.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation.js rename to ui/static/bower_components/jquery/src/manipulation.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/_evalUrl.js b/ui/static/bower_components/jquery/src/manipulation/_evalUrl.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/_evalUrl.js rename to ui/static/bower_components/jquery/src/manipulation/_evalUrl.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/buildFragment.js b/ui/static/bower_components/jquery/src/manipulation/buildFragment.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/buildFragment.js rename to ui/static/bower_components/jquery/src/manipulation/buildFragment.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/getAll.js b/ui/static/bower_components/jquery/src/manipulation/getAll.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/getAll.js rename to ui/static/bower_components/jquery/src/manipulation/getAll.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/setGlobalEval.js b/ui/static/bower_components/jquery/src/manipulation/setGlobalEval.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/setGlobalEval.js rename to ui/static/bower_components/jquery/src/manipulation/setGlobalEval.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/support.js b/ui/static/bower_components/jquery/src/manipulation/support.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/support.js rename to ui/static/bower_components/jquery/src/manipulation/support.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rcheckableType.js b/ui/static/bower_components/jquery/src/manipulation/var/rcheckableType.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rcheckableType.js rename to ui/static/bower_components/jquery/src/manipulation/var/rcheckableType.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rscriptType.js b/ui/static/bower_components/jquery/src/manipulation/var/rscriptType.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rscriptType.js rename to ui/static/bower_components/jquery/src/manipulation/var/rscriptType.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rtagName.js b/ui/static/bower_components/jquery/src/manipulation/var/rtagName.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/var/rtagName.js rename to ui/static/bower_components/jquery/src/manipulation/var/rtagName.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/manipulation/wrapMap.js b/ui/static/bower_components/jquery/src/manipulation/wrapMap.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/manipulation/wrapMap.js rename to ui/static/bower_components/jquery/src/manipulation/wrapMap.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/offset.js b/ui/static/bower_components/jquery/src/offset.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/offset.js rename to ui/static/bower_components/jquery/src/offset.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/outro.js b/ui/static/bower_components/jquery/src/outro.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/outro.js rename to ui/static/bower_components/jquery/src/outro.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/queue.js b/ui/static/bower_components/jquery/src/queue.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/queue.js rename to ui/static/bower_components/jquery/src/queue.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/queue/delay.js b/ui/static/bower_components/jquery/src/queue/delay.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/queue/delay.js rename to ui/static/bower_components/jquery/src/queue/delay.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/selector-native.js b/ui/static/bower_components/jquery/src/selector-native.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/selector-native.js rename to ui/static/bower_components/jquery/src/selector-native.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/selector-sizzle.js b/ui/static/bower_components/jquery/src/selector-sizzle.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/selector-sizzle.js rename to ui/static/bower_components/jquery/src/selector-sizzle.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/selector.js b/ui/static/bower_components/jquery/src/selector.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/selector.js rename to ui/static/bower_components/jquery/src/selector.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/serialize.js b/ui/static/bower_components/jquery/src/serialize.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/serialize.js rename to ui/static/bower_components/jquery/src/serialize.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/traversing.js b/ui/static/bower_components/jquery/src/traversing.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/traversing.js rename to ui/static/bower_components/jquery/src/traversing.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/traversing/findFilter.js b/ui/static/bower_components/jquery/src/traversing/findFilter.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/traversing/findFilter.js rename to ui/static/bower_components/jquery/src/traversing/findFilter.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/traversing/var/dir.js b/ui/static/bower_components/jquery/src/traversing/var/dir.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/traversing/var/dir.js rename to ui/static/bower_components/jquery/src/traversing/var/dir.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/traversing/var/rneedsContext.js b/ui/static/bower_components/jquery/src/traversing/var/rneedsContext.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/traversing/var/rneedsContext.js rename to ui/static/bower_components/jquery/src/traversing/var/rneedsContext.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/traversing/var/siblings.js b/ui/static/bower_components/jquery/src/traversing/var/siblings.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/traversing/var/siblings.js rename to ui/static/bower_components/jquery/src/traversing/var/siblings.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/arr.js b/ui/static/bower_components/jquery/src/var/arr.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/arr.js rename to ui/static/bower_components/jquery/src/var/arr.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/class2type.js b/ui/static/bower_components/jquery/src/var/class2type.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/class2type.js rename to ui/static/bower_components/jquery/src/var/class2type.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/concat.js b/ui/static/bower_components/jquery/src/var/concat.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/concat.js rename to ui/static/bower_components/jquery/src/var/concat.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/document.js b/ui/static/bower_components/jquery/src/var/document.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/document.js rename to ui/static/bower_components/jquery/src/var/document.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/documentElement.js b/ui/static/bower_components/jquery/src/var/documentElement.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/documentElement.js rename to ui/static/bower_components/jquery/src/var/documentElement.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/hasOwn.js b/ui/static/bower_components/jquery/src/var/hasOwn.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/hasOwn.js rename to ui/static/bower_components/jquery/src/var/hasOwn.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/indexOf.js b/ui/static/bower_components/jquery/src/var/indexOf.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/indexOf.js rename to ui/static/bower_components/jquery/src/var/indexOf.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/pnum.js b/ui/static/bower_components/jquery/src/var/pnum.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/pnum.js rename to ui/static/bower_components/jquery/src/var/pnum.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/push.js b/ui/static/bower_components/jquery/src/var/push.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/push.js rename to ui/static/bower_components/jquery/src/var/push.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/rcssNum.js b/ui/static/bower_components/jquery/src/var/rcssNum.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/rcssNum.js rename to ui/static/bower_components/jquery/src/var/rcssNum.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/rnotwhite.js b/ui/static/bower_components/jquery/src/var/rnotwhite.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/rnotwhite.js rename to ui/static/bower_components/jquery/src/var/rnotwhite.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/slice.js b/ui/static/bower_components/jquery/src/var/slice.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/slice.js rename to ui/static/bower_components/jquery/src/var/slice.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/support.js b/ui/static/bower_components/jquery/src/var/support.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/support.js rename to ui/static/bower_components/jquery/src/var/support.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/var/toString.js b/ui/static/bower_components/jquery/src/var/toString.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/var/toString.js rename to ui/static/bower_components/jquery/src/var/toString.js diff --git a/themes/bootstrap/public/bower_components/jquery/src/wrap.js b/ui/static/bower_components/jquery/src/wrap.js similarity index 100% rename from themes/bootstrap/public/bower_components/jquery/src/wrap.js rename to ui/static/bower_components/jquery/src/wrap.js diff --git a/themes/bootstrap/public/bower_components/moment/.bower.json b/ui/static/bower_components/moment/.bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/moment/.bower.json rename to ui/static/bower_components/moment/.bower.json diff --git a/themes/bootstrap/public/bower_components/moment/CHANGELOG.md b/ui/static/bower_components/moment/CHANGELOG.md similarity index 100% rename from themes/bootstrap/public/bower_components/moment/CHANGELOG.md rename to ui/static/bower_components/moment/CHANGELOG.md diff --git a/themes/bootstrap/public/bower_components/moment/LICENSE b/ui/static/bower_components/moment/LICENSE similarity index 100% rename from themes/bootstrap/public/bower_components/moment/LICENSE rename to ui/static/bower_components/moment/LICENSE diff --git a/themes/bootstrap/public/bower_components/moment/README.md b/ui/static/bower_components/moment/README.md similarity index 100% rename from themes/bootstrap/public/bower_components/moment/README.md rename to ui/static/bower_components/moment/README.md diff --git a/themes/bootstrap/public/bower_components/moment/bower.json b/ui/static/bower_components/moment/bower.json similarity index 100% rename from themes/bootstrap/public/bower_components/moment/bower.json rename to ui/static/bower_components/moment/bower.json diff --git a/themes/bootstrap/public/bower_components/moment/locale/af.js b/ui/static/bower_components/moment/locale/af.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/af.js rename to ui/static/bower_components/moment/locale/af.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar-dz.js b/ui/static/bower_components/moment/locale/ar-dz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar-dz.js rename to ui/static/bower_components/moment/locale/ar-dz.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar-ly.js b/ui/static/bower_components/moment/locale/ar-ly.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar-ly.js rename to ui/static/bower_components/moment/locale/ar-ly.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar-ma.js b/ui/static/bower_components/moment/locale/ar-ma.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar-ma.js rename to ui/static/bower_components/moment/locale/ar-ma.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar-sa.js b/ui/static/bower_components/moment/locale/ar-sa.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar-sa.js rename to ui/static/bower_components/moment/locale/ar-sa.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar-tn.js b/ui/static/bower_components/moment/locale/ar-tn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar-tn.js rename to ui/static/bower_components/moment/locale/ar-tn.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ar.js b/ui/static/bower_components/moment/locale/ar.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ar.js rename to ui/static/bower_components/moment/locale/ar.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/az.js b/ui/static/bower_components/moment/locale/az.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/az.js rename to ui/static/bower_components/moment/locale/az.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/be.js b/ui/static/bower_components/moment/locale/be.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/be.js rename to ui/static/bower_components/moment/locale/be.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/bg.js b/ui/static/bower_components/moment/locale/bg.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/bg.js rename to ui/static/bower_components/moment/locale/bg.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/bn.js b/ui/static/bower_components/moment/locale/bn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/bn.js rename to ui/static/bower_components/moment/locale/bn.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/bo.js b/ui/static/bower_components/moment/locale/bo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/bo.js rename to ui/static/bower_components/moment/locale/bo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/br.js b/ui/static/bower_components/moment/locale/br.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/br.js rename to ui/static/bower_components/moment/locale/br.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/bs.js b/ui/static/bower_components/moment/locale/bs.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/bs.js rename to ui/static/bower_components/moment/locale/bs.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ca.js b/ui/static/bower_components/moment/locale/ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ca.js rename to ui/static/bower_components/moment/locale/ca.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/cs.js b/ui/static/bower_components/moment/locale/cs.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/cs.js rename to ui/static/bower_components/moment/locale/cs.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/cv.js b/ui/static/bower_components/moment/locale/cv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/cv.js rename to ui/static/bower_components/moment/locale/cv.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/cy.js b/ui/static/bower_components/moment/locale/cy.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/cy.js rename to ui/static/bower_components/moment/locale/cy.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/da.js b/ui/static/bower_components/moment/locale/da.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/da.js rename to ui/static/bower_components/moment/locale/da.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/de-at.js b/ui/static/bower_components/moment/locale/de-at.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/de-at.js rename to ui/static/bower_components/moment/locale/de-at.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/de.js b/ui/static/bower_components/moment/locale/de.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/de.js rename to ui/static/bower_components/moment/locale/de.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/dv.js b/ui/static/bower_components/moment/locale/dv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/dv.js rename to ui/static/bower_components/moment/locale/dv.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/el.js b/ui/static/bower_components/moment/locale/el.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/el.js rename to ui/static/bower_components/moment/locale/el.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/en-au.js b/ui/static/bower_components/moment/locale/en-au.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/en-au.js rename to ui/static/bower_components/moment/locale/en-au.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/en-ca.js b/ui/static/bower_components/moment/locale/en-ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/en-ca.js rename to ui/static/bower_components/moment/locale/en-ca.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/en-gb.js b/ui/static/bower_components/moment/locale/en-gb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/en-gb.js rename to ui/static/bower_components/moment/locale/en-gb.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/en-ie.js b/ui/static/bower_components/moment/locale/en-ie.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/en-ie.js rename to ui/static/bower_components/moment/locale/en-ie.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/en-nz.js b/ui/static/bower_components/moment/locale/en-nz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/en-nz.js rename to ui/static/bower_components/moment/locale/en-nz.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/eo.js b/ui/static/bower_components/moment/locale/eo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/eo.js rename to ui/static/bower_components/moment/locale/eo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/es-do.js b/ui/static/bower_components/moment/locale/es-do.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/es-do.js rename to ui/static/bower_components/moment/locale/es-do.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/es.js b/ui/static/bower_components/moment/locale/es.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/es.js rename to ui/static/bower_components/moment/locale/es.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/et.js b/ui/static/bower_components/moment/locale/et.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/et.js rename to ui/static/bower_components/moment/locale/et.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/eu.js b/ui/static/bower_components/moment/locale/eu.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/eu.js rename to ui/static/bower_components/moment/locale/eu.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fa.js b/ui/static/bower_components/moment/locale/fa.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fa.js rename to ui/static/bower_components/moment/locale/fa.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fi.js b/ui/static/bower_components/moment/locale/fi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fi.js rename to ui/static/bower_components/moment/locale/fi.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fo.js b/ui/static/bower_components/moment/locale/fo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fo.js rename to ui/static/bower_components/moment/locale/fo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fr-ca.js b/ui/static/bower_components/moment/locale/fr-ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fr-ca.js rename to ui/static/bower_components/moment/locale/fr-ca.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fr-ch.js b/ui/static/bower_components/moment/locale/fr-ch.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fr-ch.js rename to ui/static/bower_components/moment/locale/fr-ch.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fr.js b/ui/static/bower_components/moment/locale/fr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fr.js rename to ui/static/bower_components/moment/locale/fr.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/fy.js b/ui/static/bower_components/moment/locale/fy.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/fy.js rename to ui/static/bower_components/moment/locale/fy.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/gd.js b/ui/static/bower_components/moment/locale/gd.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/gd.js rename to ui/static/bower_components/moment/locale/gd.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/gl.js b/ui/static/bower_components/moment/locale/gl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/gl.js rename to ui/static/bower_components/moment/locale/gl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/he.js b/ui/static/bower_components/moment/locale/he.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/he.js rename to ui/static/bower_components/moment/locale/he.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/hi.js b/ui/static/bower_components/moment/locale/hi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/hi.js rename to ui/static/bower_components/moment/locale/hi.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/hr.js b/ui/static/bower_components/moment/locale/hr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/hr.js rename to ui/static/bower_components/moment/locale/hr.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/hu.js b/ui/static/bower_components/moment/locale/hu.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/hu.js rename to ui/static/bower_components/moment/locale/hu.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/hy-am.js b/ui/static/bower_components/moment/locale/hy-am.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/hy-am.js rename to ui/static/bower_components/moment/locale/hy-am.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/id.js b/ui/static/bower_components/moment/locale/id.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/id.js rename to ui/static/bower_components/moment/locale/id.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/is.js b/ui/static/bower_components/moment/locale/is.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/is.js rename to ui/static/bower_components/moment/locale/is.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/it.js b/ui/static/bower_components/moment/locale/it.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/it.js rename to ui/static/bower_components/moment/locale/it.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ja.js b/ui/static/bower_components/moment/locale/ja.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ja.js rename to ui/static/bower_components/moment/locale/ja.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/jv.js b/ui/static/bower_components/moment/locale/jv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/jv.js rename to ui/static/bower_components/moment/locale/jv.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ka.js b/ui/static/bower_components/moment/locale/ka.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ka.js rename to ui/static/bower_components/moment/locale/ka.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/kk.js b/ui/static/bower_components/moment/locale/kk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/kk.js rename to ui/static/bower_components/moment/locale/kk.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/km.js b/ui/static/bower_components/moment/locale/km.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/km.js rename to ui/static/bower_components/moment/locale/km.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ko.js b/ui/static/bower_components/moment/locale/ko.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ko.js rename to ui/static/bower_components/moment/locale/ko.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ky.js b/ui/static/bower_components/moment/locale/ky.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ky.js rename to ui/static/bower_components/moment/locale/ky.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/lb.js b/ui/static/bower_components/moment/locale/lb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/lb.js rename to ui/static/bower_components/moment/locale/lb.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/lo.js b/ui/static/bower_components/moment/locale/lo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/lo.js rename to ui/static/bower_components/moment/locale/lo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/lt.js b/ui/static/bower_components/moment/locale/lt.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/lt.js rename to ui/static/bower_components/moment/locale/lt.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/lv.js b/ui/static/bower_components/moment/locale/lv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/lv.js rename to ui/static/bower_components/moment/locale/lv.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/me.js b/ui/static/bower_components/moment/locale/me.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/me.js rename to ui/static/bower_components/moment/locale/me.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/mi.js b/ui/static/bower_components/moment/locale/mi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/mi.js rename to ui/static/bower_components/moment/locale/mi.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/mk.js b/ui/static/bower_components/moment/locale/mk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/mk.js rename to ui/static/bower_components/moment/locale/mk.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ml.js b/ui/static/bower_components/moment/locale/ml.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ml.js rename to ui/static/bower_components/moment/locale/ml.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/mr.js b/ui/static/bower_components/moment/locale/mr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/mr.js rename to ui/static/bower_components/moment/locale/mr.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ms-my.js b/ui/static/bower_components/moment/locale/ms-my.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ms-my.js rename to ui/static/bower_components/moment/locale/ms-my.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ms.js b/ui/static/bower_components/moment/locale/ms.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ms.js rename to ui/static/bower_components/moment/locale/ms.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/my.js b/ui/static/bower_components/moment/locale/my.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/my.js rename to ui/static/bower_components/moment/locale/my.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/nb.js b/ui/static/bower_components/moment/locale/nb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/nb.js rename to ui/static/bower_components/moment/locale/nb.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ne.js b/ui/static/bower_components/moment/locale/ne.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ne.js rename to ui/static/bower_components/moment/locale/ne.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/nl-be.js b/ui/static/bower_components/moment/locale/nl-be.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/nl-be.js rename to ui/static/bower_components/moment/locale/nl-be.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/nl.js b/ui/static/bower_components/moment/locale/nl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/nl.js rename to ui/static/bower_components/moment/locale/nl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/nn.js b/ui/static/bower_components/moment/locale/nn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/nn.js rename to ui/static/bower_components/moment/locale/nn.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/pa-in.js b/ui/static/bower_components/moment/locale/pa-in.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/pa-in.js rename to ui/static/bower_components/moment/locale/pa-in.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/pl.js b/ui/static/bower_components/moment/locale/pl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/pl.js rename to ui/static/bower_components/moment/locale/pl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/pt-br.js b/ui/static/bower_components/moment/locale/pt-br.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/pt-br.js rename to ui/static/bower_components/moment/locale/pt-br.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/pt.js b/ui/static/bower_components/moment/locale/pt.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/pt.js rename to ui/static/bower_components/moment/locale/pt.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ro.js b/ui/static/bower_components/moment/locale/ro.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ro.js rename to ui/static/bower_components/moment/locale/ro.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ru.js b/ui/static/bower_components/moment/locale/ru.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ru.js rename to ui/static/bower_components/moment/locale/ru.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/se.js b/ui/static/bower_components/moment/locale/se.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/se.js rename to ui/static/bower_components/moment/locale/se.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/si.js b/ui/static/bower_components/moment/locale/si.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/si.js rename to ui/static/bower_components/moment/locale/si.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sk.js b/ui/static/bower_components/moment/locale/sk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sk.js rename to ui/static/bower_components/moment/locale/sk.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sl.js b/ui/static/bower_components/moment/locale/sl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sl.js rename to ui/static/bower_components/moment/locale/sl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sq.js b/ui/static/bower_components/moment/locale/sq.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sq.js rename to ui/static/bower_components/moment/locale/sq.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sr-cyrl.js b/ui/static/bower_components/moment/locale/sr-cyrl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sr-cyrl.js rename to ui/static/bower_components/moment/locale/sr-cyrl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sr.js b/ui/static/bower_components/moment/locale/sr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sr.js rename to ui/static/bower_components/moment/locale/sr.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ss.js b/ui/static/bower_components/moment/locale/ss.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ss.js rename to ui/static/bower_components/moment/locale/ss.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sv.js b/ui/static/bower_components/moment/locale/sv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sv.js rename to ui/static/bower_components/moment/locale/sv.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/sw.js b/ui/static/bower_components/moment/locale/sw.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/sw.js rename to ui/static/bower_components/moment/locale/sw.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/ta.js b/ui/static/bower_components/moment/locale/ta.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/ta.js rename to ui/static/bower_components/moment/locale/ta.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/te.js b/ui/static/bower_components/moment/locale/te.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/te.js rename to ui/static/bower_components/moment/locale/te.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tet.js b/ui/static/bower_components/moment/locale/tet.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tet.js rename to ui/static/bower_components/moment/locale/tet.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/th.js b/ui/static/bower_components/moment/locale/th.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/th.js rename to ui/static/bower_components/moment/locale/th.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tl-ph.js b/ui/static/bower_components/moment/locale/tl-ph.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tl-ph.js rename to ui/static/bower_components/moment/locale/tl-ph.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tlh.js b/ui/static/bower_components/moment/locale/tlh.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tlh.js rename to ui/static/bower_components/moment/locale/tlh.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tr.js b/ui/static/bower_components/moment/locale/tr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tr.js rename to ui/static/bower_components/moment/locale/tr.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tzl.js b/ui/static/bower_components/moment/locale/tzl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tzl.js rename to ui/static/bower_components/moment/locale/tzl.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tzm-latn.js b/ui/static/bower_components/moment/locale/tzm-latn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tzm-latn.js rename to ui/static/bower_components/moment/locale/tzm-latn.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/tzm.js b/ui/static/bower_components/moment/locale/tzm.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/tzm.js rename to ui/static/bower_components/moment/locale/tzm.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/uk.js b/ui/static/bower_components/moment/locale/uk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/uk.js rename to ui/static/bower_components/moment/locale/uk.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/uz.js b/ui/static/bower_components/moment/locale/uz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/uz.js rename to ui/static/bower_components/moment/locale/uz.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/vi.js b/ui/static/bower_components/moment/locale/vi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/vi.js rename to ui/static/bower_components/moment/locale/vi.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/x-pseudo.js b/ui/static/bower_components/moment/locale/x-pseudo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/x-pseudo.js rename to ui/static/bower_components/moment/locale/x-pseudo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/yo.js b/ui/static/bower_components/moment/locale/yo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/yo.js rename to ui/static/bower_components/moment/locale/yo.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/zh-cn.js b/ui/static/bower_components/moment/locale/zh-cn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/zh-cn.js rename to ui/static/bower_components/moment/locale/zh-cn.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/zh-hk.js b/ui/static/bower_components/moment/locale/zh-hk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/zh-hk.js rename to ui/static/bower_components/moment/locale/zh-hk.js diff --git a/themes/bootstrap/public/bower_components/moment/locale/zh-tw.js b/ui/static/bower_components/moment/locale/zh-tw.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/locale/zh-tw.js rename to ui/static/bower_components/moment/locale/zh-tw.js diff --git a/themes/bootstrap/public/bower_components/moment/min/locales.js b/ui/static/bower_components/moment/min/locales.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/locales.js rename to ui/static/bower_components/moment/min/locales.js diff --git a/themes/bootstrap/public/bower_components/moment/min/locales.min.js b/ui/static/bower_components/moment/min/locales.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/locales.min.js rename to ui/static/bower_components/moment/min/locales.min.js diff --git a/themes/bootstrap/public/bower_components/moment/min/moment-with-locales.js b/ui/static/bower_components/moment/min/moment-with-locales.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/moment-with-locales.js rename to ui/static/bower_components/moment/min/moment-with-locales.js diff --git a/themes/bootstrap/public/bower_components/moment/min/moment-with-locales.min.js b/ui/static/bower_components/moment/min/moment-with-locales.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/moment-with-locales.min.js rename to ui/static/bower_components/moment/min/moment-with-locales.min.js diff --git a/themes/bootstrap/public/bower_components/moment/min/moment.min.js b/ui/static/bower_components/moment/min/moment.min.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/moment.min.js rename to ui/static/bower_components/moment/min/moment.min.js diff --git a/themes/bootstrap/public/bower_components/moment/min/tests.js b/ui/static/bower_components/moment/min/tests.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/min/tests.js rename to ui/static/bower_components/moment/min/tests.js diff --git a/themes/bootstrap/public/bower_components/moment/moment.d.ts b/ui/static/bower_components/moment/moment.d.ts similarity index 100% rename from themes/bootstrap/public/bower_components/moment/moment.d.ts rename to ui/static/bower_components/moment/moment.d.ts diff --git a/themes/bootstrap/public/bower_components/moment/moment.js b/ui/static/bower_components/moment/moment.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/moment.js rename to ui/static/bower_components/moment/moment.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/check-overflow.js b/ui/static/bower_components/moment/src/lib/create/check-overflow.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/check-overflow.js rename to ui/static/bower_components/moment/src/lib/create/check-overflow.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/date-from-array.js b/ui/static/bower_components/moment/src/lib/create/date-from-array.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/date-from-array.js rename to ui/static/bower_components/moment/src/lib/create/date-from-array.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-anything.js b/ui/static/bower_components/moment/src/lib/create/from-anything.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-anything.js rename to ui/static/bower_components/moment/src/lib/create/from-anything.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-array.js b/ui/static/bower_components/moment/src/lib/create/from-array.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-array.js rename to ui/static/bower_components/moment/src/lib/create/from-array.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-object.js b/ui/static/bower_components/moment/src/lib/create/from-object.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-object.js rename to ui/static/bower_components/moment/src/lib/create/from-object.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-string-and-array.js b/ui/static/bower_components/moment/src/lib/create/from-string-and-array.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-string-and-array.js rename to ui/static/bower_components/moment/src/lib/create/from-string-and-array.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-string-and-format.js b/ui/static/bower_components/moment/src/lib/create/from-string-and-format.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-string-and-format.js rename to ui/static/bower_components/moment/src/lib/create/from-string-and-format.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/from-string.js b/ui/static/bower_components/moment/src/lib/create/from-string.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/from-string.js rename to ui/static/bower_components/moment/src/lib/create/from-string.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/local.js b/ui/static/bower_components/moment/src/lib/create/local.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/local.js rename to ui/static/bower_components/moment/src/lib/create/local.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/parsing-flags.js b/ui/static/bower_components/moment/src/lib/create/parsing-flags.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/parsing-flags.js rename to ui/static/bower_components/moment/src/lib/create/parsing-flags.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/utc.js b/ui/static/bower_components/moment/src/lib/create/utc.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/utc.js rename to ui/static/bower_components/moment/src/lib/create/utc.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/create/valid.js b/ui/static/bower_components/moment/src/lib/create/valid.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/create/valid.js rename to ui/static/bower_components/moment/src/lib/create/valid.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/abs.js b/ui/static/bower_components/moment/src/lib/duration/abs.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/abs.js rename to ui/static/bower_components/moment/src/lib/duration/abs.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/add-subtract.js b/ui/static/bower_components/moment/src/lib/duration/add-subtract.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/add-subtract.js rename to ui/static/bower_components/moment/src/lib/duration/add-subtract.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/as.js b/ui/static/bower_components/moment/src/lib/duration/as.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/as.js rename to ui/static/bower_components/moment/src/lib/duration/as.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/bubble.js b/ui/static/bower_components/moment/src/lib/duration/bubble.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/bubble.js rename to ui/static/bower_components/moment/src/lib/duration/bubble.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/constructor.js b/ui/static/bower_components/moment/src/lib/duration/constructor.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/constructor.js rename to ui/static/bower_components/moment/src/lib/duration/constructor.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/create.js b/ui/static/bower_components/moment/src/lib/duration/create.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/create.js rename to ui/static/bower_components/moment/src/lib/duration/create.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/duration.js b/ui/static/bower_components/moment/src/lib/duration/duration.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/duration.js rename to ui/static/bower_components/moment/src/lib/duration/duration.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/get.js b/ui/static/bower_components/moment/src/lib/duration/get.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/get.js rename to ui/static/bower_components/moment/src/lib/duration/get.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/humanize.js b/ui/static/bower_components/moment/src/lib/duration/humanize.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/humanize.js rename to ui/static/bower_components/moment/src/lib/duration/humanize.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/iso-string.js b/ui/static/bower_components/moment/src/lib/duration/iso-string.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/iso-string.js rename to ui/static/bower_components/moment/src/lib/duration/iso-string.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/duration/prototype.js b/ui/static/bower_components/moment/src/lib/duration/prototype.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/duration/prototype.js rename to ui/static/bower_components/moment/src/lib/duration/prototype.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/format/format.js b/ui/static/bower_components/moment/src/lib/format/format.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/format/format.js rename to ui/static/bower_components/moment/src/lib/format/format.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/base-config.js b/ui/static/bower_components/moment/src/lib/locale/base-config.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/base-config.js rename to ui/static/bower_components/moment/src/lib/locale/base-config.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/calendar.js b/ui/static/bower_components/moment/src/lib/locale/calendar.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/calendar.js rename to ui/static/bower_components/moment/src/lib/locale/calendar.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/constructor.js b/ui/static/bower_components/moment/src/lib/locale/constructor.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/constructor.js rename to ui/static/bower_components/moment/src/lib/locale/constructor.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/en.js b/ui/static/bower_components/moment/src/lib/locale/en.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/en.js rename to ui/static/bower_components/moment/src/lib/locale/en.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/formats.js b/ui/static/bower_components/moment/src/lib/locale/formats.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/formats.js rename to ui/static/bower_components/moment/src/lib/locale/formats.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/invalid.js b/ui/static/bower_components/moment/src/lib/locale/invalid.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/invalid.js rename to ui/static/bower_components/moment/src/lib/locale/invalid.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/lists.js b/ui/static/bower_components/moment/src/lib/locale/lists.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/lists.js rename to ui/static/bower_components/moment/src/lib/locale/lists.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/locale.js b/ui/static/bower_components/moment/src/lib/locale/locale.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/locale.js rename to ui/static/bower_components/moment/src/lib/locale/locale.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/locales.js b/ui/static/bower_components/moment/src/lib/locale/locales.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/locales.js rename to ui/static/bower_components/moment/src/lib/locale/locales.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/ordinal.js b/ui/static/bower_components/moment/src/lib/locale/ordinal.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/ordinal.js rename to ui/static/bower_components/moment/src/lib/locale/ordinal.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/pre-post-format.js b/ui/static/bower_components/moment/src/lib/locale/pre-post-format.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/pre-post-format.js rename to ui/static/bower_components/moment/src/lib/locale/pre-post-format.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/prototype.js b/ui/static/bower_components/moment/src/lib/locale/prototype.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/prototype.js rename to ui/static/bower_components/moment/src/lib/locale/prototype.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/relative.js b/ui/static/bower_components/moment/src/lib/locale/relative.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/relative.js rename to ui/static/bower_components/moment/src/lib/locale/relative.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/locale/set.js b/ui/static/bower_components/moment/src/lib/locale/set.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/locale/set.js rename to ui/static/bower_components/moment/src/lib/locale/set.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/add-subtract.js b/ui/static/bower_components/moment/src/lib/moment/add-subtract.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/add-subtract.js rename to ui/static/bower_components/moment/src/lib/moment/add-subtract.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/calendar.js b/ui/static/bower_components/moment/src/lib/moment/calendar.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/calendar.js rename to ui/static/bower_components/moment/src/lib/moment/calendar.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/clone.js b/ui/static/bower_components/moment/src/lib/moment/clone.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/clone.js rename to ui/static/bower_components/moment/src/lib/moment/clone.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/compare.js b/ui/static/bower_components/moment/src/lib/moment/compare.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/compare.js rename to ui/static/bower_components/moment/src/lib/moment/compare.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/constructor.js b/ui/static/bower_components/moment/src/lib/moment/constructor.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/constructor.js rename to ui/static/bower_components/moment/src/lib/moment/constructor.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/creation-data.js b/ui/static/bower_components/moment/src/lib/moment/creation-data.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/creation-data.js rename to ui/static/bower_components/moment/src/lib/moment/creation-data.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/diff.js b/ui/static/bower_components/moment/src/lib/moment/diff.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/diff.js rename to ui/static/bower_components/moment/src/lib/moment/diff.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/format.js b/ui/static/bower_components/moment/src/lib/moment/format.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/format.js rename to ui/static/bower_components/moment/src/lib/moment/format.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/from.js b/ui/static/bower_components/moment/src/lib/moment/from.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/from.js rename to ui/static/bower_components/moment/src/lib/moment/from.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/get-set.js b/ui/static/bower_components/moment/src/lib/moment/get-set.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/get-set.js rename to ui/static/bower_components/moment/src/lib/moment/get-set.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/locale.js b/ui/static/bower_components/moment/src/lib/moment/locale.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/locale.js rename to ui/static/bower_components/moment/src/lib/moment/locale.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/min-max.js b/ui/static/bower_components/moment/src/lib/moment/min-max.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/min-max.js rename to ui/static/bower_components/moment/src/lib/moment/min-max.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/moment.js b/ui/static/bower_components/moment/src/lib/moment/moment.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/moment.js rename to ui/static/bower_components/moment/src/lib/moment/moment.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/now.js b/ui/static/bower_components/moment/src/lib/moment/now.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/now.js rename to ui/static/bower_components/moment/src/lib/moment/now.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/prototype.js b/ui/static/bower_components/moment/src/lib/moment/prototype.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/prototype.js rename to ui/static/bower_components/moment/src/lib/moment/prototype.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/start-end-of.js b/ui/static/bower_components/moment/src/lib/moment/start-end-of.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/start-end-of.js rename to ui/static/bower_components/moment/src/lib/moment/start-end-of.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/to-type.js b/ui/static/bower_components/moment/src/lib/moment/to-type.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/to-type.js rename to ui/static/bower_components/moment/src/lib/moment/to-type.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/to.js b/ui/static/bower_components/moment/src/lib/moment/to.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/to.js rename to ui/static/bower_components/moment/src/lib/moment/to.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/moment/valid.js b/ui/static/bower_components/moment/src/lib/moment/valid.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/moment/valid.js rename to ui/static/bower_components/moment/src/lib/moment/valid.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/parse/regex.js b/ui/static/bower_components/moment/src/lib/parse/regex.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/parse/regex.js rename to ui/static/bower_components/moment/src/lib/parse/regex.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/parse/token.js b/ui/static/bower_components/moment/src/lib/parse/token.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/parse/token.js rename to ui/static/bower_components/moment/src/lib/parse/token.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/aliases.js b/ui/static/bower_components/moment/src/lib/units/aliases.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/aliases.js rename to ui/static/bower_components/moment/src/lib/units/aliases.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/constants.js b/ui/static/bower_components/moment/src/lib/units/constants.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/constants.js rename to ui/static/bower_components/moment/src/lib/units/constants.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-month.js b/ui/static/bower_components/moment/src/lib/units/day-of-month.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-month.js rename to ui/static/bower_components/moment/src/lib/units/day-of-month.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-week.js b/ui/static/bower_components/moment/src/lib/units/day-of-week.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-week.js rename to ui/static/bower_components/moment/src/lib/units/day-of-week.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-year.js b/ui/static/bower_components/moment/src/lib/units/day-of-year.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/day-of-year.js rename to ui/static/bower_components/moment/src/lib/units/day-of-year.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/hour.js b/ui/static/bower_components/moment/src/lib/units/hour.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/hour.js rename to ui/static/bower_components/moment/src/lib/units/hour.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/millisecond.js b/ui/static/bower_components/moment/src/lib/units/millisecond.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/millisecond.js rename to ui/static/bower_components/moment/src/lib/units/millisecond.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/minute.js b/ui/static/bower_components/moment/src/lib/units/minute.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/minute.js rename to ui/static/bower_components/moment/src/lib/units/minute.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/month.js b/ui/static/bower_components/moment/src/lib/units/month.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/month.js rename to ui/static/bower_components/moment/src/lib/units/month.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/offset.js b/ui/static/bower_components/moment/src/lib/units/offset.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/offset.js rename to ui/static/bower_components/moment/src/lib/units/offset.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/priorities.js b/ui/static/bower_components/moment/src/lib/units/priorities.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/priorities.js rename to ui/static/bower_components/moment/src/lib/units/priorities.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/quarter.js b/ui/static/bower_components/moment/src/lib/units/quarter.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/quarter.js rename to ui/static/bower_components/moment/src/lib/units/quarter.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/second.js b/ui/static/bower_components/moment/src/lib/units/second.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/second.js rename to ui/static/bower_components/moment/src/lib/units/second.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/timestamp.js b/ui/static/bower_components/moment/src/lib/units/timestamp.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/timestamp.js rename to ui/static/bower_components/moment/src/lib/units/timestamp.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/timezone.js b/ui/static/bower_components/moment/src/lib/units/timezone.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/timezone.js rename to ui/static/bower_components/moment/src/lib/units/timezone.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/units.js b/ui/static/bower_components/moment/src/lib/units/units.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/units.js rename to ui/static/bower_components/moment/src/lib/units/units.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/week-calendar-utils.js b/ui/static/bower_components/moment/src/lib/units/week-calendar-utils.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/week-calendar-utils.js rename to ui/static/bower_components/moment/src/lib/units/week-calendar-utils.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/week-year.js b/ui/static/bower_components/moment/src/lib/units/week-year.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/week-year.js rename to ui/static/bower_components/moment/src/lib/units/week-year.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/week.js b/ui/static/bower_components/moment/src/lib/units/week.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/week.js rename to ui/static/bower_components/moment/src/lib/units/week.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/units/year.js b/ui/static/bower_components/moment/src/lib/units/year.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/units/year.js rename to ui/static/bower_components/moment/src/lib/units/year.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-ceil.js b/ui/static/bower_components/moment/src/lib/utils/abs-ceil.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-ceil.js rename to ui/static/bower_components/moment/src/lib/utils/abs-ceil.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-floor.js b/ui/static/bower_components/moment/src/lib/utils/abs-floor.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-floor.js rename to ui/static/bower_components/moment/src/lib/utils/abs-floor.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-round.js b/ui/static/bower_components/moment/src/lib/utils/abs-round.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/abs-round.js rename to ui/static/bower_components/moment/src/lib/utils/abs-round.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/compare-arrays.js b/ui/static/bower_components/moment/src/lib/utils/compare-arrays.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/compare-arrays.js rename to ui/static/bower_components/moment/src/lib/utils/compare-arrays.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/defaults.js b/ui/static/bower_components/moment/src/lib/utils/defaults.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/defaults.js rename to ui/static/bower_components/moment/src/lib/utils/defaults.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/deprecate.js b/ui/static/bower_components/moment/src/lib/utils/deprecate.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/deprecate.js rename to ui/static/bower_components/moment/src/lib/utils/deprecate.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/extend.js b/ui/static/bower_components/moment/src/lib/utils/extend.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/extend.js rename to ui/static/bower_components/moment/src/lib/utils/extend.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/has-own-prop.js b/ui/static/bower_components/moment/src/lib/utils/has-own-prop.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/has-own-prop.js rename to ui/static/bower_components/moment/src/lib/utils/has-own-prop.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/hooks.js b/ui/static/bower_components/moment/src/lib/utils/hooks.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/hooks.js rename to ui/static/bower_components/moment/src/lib/utils/hooks.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/index-of.js b/ui/static/bower_components/moment/src/lib/utils/index-of.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/index-of.js rename to ui/static/bower_components/moment/src/lib/utils/index-of.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-array.js b/ui/static/bower_components/moment/src/lib/utils/is-array.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-array.js rename to ui/static/bower_components/moment/src/lib/utils/is-array.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-date.js b/ui/static/bower_components/moment/src/lib/utils/is-date.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-date.js rename to ui/static/bower_components/moment/src/lib/utils/is-date.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-function.js b/ui/static/bower_components/moment/src/lib/utils/is-function.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-function.js rename to ui/static/bower_components/moment/src/lib/utils/is-function.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-number.js b/ui/static/bower_components/moment/src/lib/utils/is-number.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-number.js rename to ui/static/bower_components/moment/src/lib/utils/is-number.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-object-empty.js b/ui/static/bower_components/moment/src/lib/utils/is-object-empty.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-object-empty.js rename to ui/static/bower_components/moment/src/lib/utils/is-object-empty.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-object.js b/ui/static/bower_components/moment/src/lib/utils/is-object.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-object.js rename to ui/static/bower_components/moment/src/lib/utils/is-object.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/is-undefined.js b/ui/static/bower_components/moment/src/lib/utils/is-undefined.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/is-undefined.js rename to ui/static/bower_components/moment/src/lib/utils/is-undefined.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/keys.js b/ui/static/bower_components/moment/src/lib/utils/keys.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/keys.js rename to ui/static/bower_components/moment/src/lib/utils/keys.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/map.js b/ui/static/bower_components/moment/src/lib/utils/map.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/map.js rename to ui/static/bower_components/moment/src/lib/utils/map.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/some.js b/ui/static/bower_components/moment/src/lib/utils/some.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/some.js rename to ui/static/bower_components/moment/src/lib/utils/some.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/to-int.js b/ui/static/bower_components/moment/src/lib/utils/to-int.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/to-int.js rename to ui/static/bower_components/moment/src/lib/utils/to-int.js diff --git a/themes/bootstrap/public/bower_components/moment/src/lib/utils/zero-fill.js b/ui/static/bower_components/moment/src/lib/utils/zero-fill.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/lib/utils/zero-fill.js rename to ui/static/bower_components/moment/src/lib/utils/zero-fill.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/af.js b/ui/static/bower_components/moment/src/locale/af.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/af.js rename to ui/static/bower_components/moment/src/locale/af.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar-dz.js b/ui/static/bower_components/moment/src/locale/ar-dz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar-dz.js rename to ui/static/bower_components/moment/src/locale/ar-dz.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar-ly.js b/ui/static/bower_components/moment/src/locale/ar-ly.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar-ly.js rename to ui/static/bower_components/moment/src/locale/ar-ly.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar-ma.js b/ui/static/bower_components/moment/src/locale/ar-ma.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar-ma.js rename to ui/static/bower_components/moment/src/locale/ar-ma.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar-sa.js b/ui/static/bower_components/moment/src/locale/ar-sa.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar-sa.js rename to ui/static/bower_components/moment/src/locale/ar-sa.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar-tn.js b/ui/static/bower_components/moment/src/locale/ar-tn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar-tn.js rename to ui/static/bower_components/moment/src/locale/ar-tn.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ar.js b/ui/static/bower_components/moment/src/locale/ar.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ar.js rename to ui/static/bower_components/moment/src/locale/ar.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/az.js b/ui/static/bower_components/moment/src/locale/az.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/az.js rename to ui/static/bower_components/moment/src/locale/az.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/be.js b/ui/static/bower_components/moment/src/locale/be.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/be.js rename to ui/static/bower_components/moment/src/locale/be.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/bg.js b/ui/static/bower_components/moment/src/locale/bg.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/bg.js rename to ui/static/bower_components/moment/src/locale/bg.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/bn.js b/ui/static/bower_components/moment/src/locale/bn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/bn.js rename to ui/static/bower_components/moment/src/locale/bn.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/bo.js b/ui/static/bower_components/moment/src/locale/bo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/bo.js rename to ui/static/bower_components/moment/src/locale/bo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/br.js b/ui/static/bower_components/moment/src/locale/br.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/br.js rename to ui/static/bower_components/moment/src/locale/br.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/bs.js b/ui/static/bower_components/moment/src/locale/bs.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/bs.js rename to ui/static/bower_components/moment/src/locale/bs.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ca.js b/ui/static/bower_components/moment/src/locale/ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ca.js rename to ui/static/bower_components/moment/src/locale/ca.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/cs.js b/ui/static/bower_components/moment/src/locale/cs.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/cs.js rename to ui/static/bower_components/moment/src/locale/cs.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/cv.js b/ui/static/bower_components/moment/src/locale/cv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/cv.js rename to ui/static/bower_components/moment/src/locale/cv.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/cy.js b/ui/static/bower_components/moment/src/locale/cy.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/cy.js rename to ui/static/bower_components/moment/src/locale/cy.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/da.js b/ui/static/bower_components/moment/src/locale/da.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/da.js rename to ui/static/bower_components/moment/src/locale/da.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/de-at.js b/ui/static/bower_components/moment/src/locale/de-at.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/de-at.js rename to ui/static/bower_components/moment/src/locale/de-at.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/de.js b/ui/static/bower_components/moment/src/locale/de.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/de.js rename to ui/static/bower_components/moment/src/locale/de.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/dv.js b/ui/static/bower_components/moment/src/locale/dv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/dv.js rename to ui/static/bower_components/moment/src/locale/dv.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/el.js b/ui/static/bower_components/moment/src/locale/el.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/el.js rename to ui/static/bower_components/moment/src/locale/el.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/en-au.js b/ui/static/bower_components/moment/src/locale/en-au.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/en-au.js rename to ui/static/bower_components/moment/src/locale/en-au.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/en-ca.js b/ui/static/bower_components/moment/src/locale/en-ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/en-ca.js rename to ui/static/bower_components/moment/src/locale/en-ca.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/en-gb.js b/ui/static/bower_components/moment/src/locale/en-gb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/en-gb.js rename to ui/static/bower_components/moment/src/locale/en-gb.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/en-ie.js b/ui/static/bower_components/moment/src/locale/en-ie.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/en-ie.js rename to ui/static/bower_components/moment/src/locale/en-ie.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/en-nz.js b/ui/static/bower_components/moment/src/locale/en-nz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/en-nz.js rename to ui/static/bower_components/moment/src/locale/en-nz.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/eo.js b/ui/static/bower_components/moment/src/locale/eo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/eo.js rename to ui/static/bower_components/moment/src/locale/eo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/es-do.js b/ui/static/bower_components/moment/src/locale/es-do.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/es-do.js rename to ui/static/bower_components/moment/src/locale/es-do.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/es.js b/ui/static/bower_components/moment/src/locale/es.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/es.js rename to ui/static/bower_components/moment/src/locale/es.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/et.js b/ui/static/bower_components/moment/src/locale/et.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/et.js rename to ui/static/bower_components/moment/src/locale/et.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/eu.js b/ui/static/bower_components/moment/src/locale/eu.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/eu.js rename to ui/static/bower_components/moment/src/locale/eu.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fa.js b/ui/static/bower_components/moment/src/locale/fa.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fa.js rename to ui/static/bower_components/moment/src/locale/fa.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fi.js b/ui/static/bower_components/moment/src/locale/fi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fi.js rename to ui/static/bower_components/moment/src/locale/fi.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fo.js b/ui/static/bower_components/moment/src/locale/fo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fo.js rename to ui/static/bower_components/moment/src/locale/fo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fr-ca.js b/ui/static/bower_components/moment/src/locale/fr-ca.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fr-ca.js rename to ui/static/bower_components/moment/src/locale/fr-ca.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fr-ch.js b/ui/static/bower_components/moment/src/locale/fr-ch.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fr-ch.js rename to ui/static/bower_components/moment/src/locale/fr-ch.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fr.js b/ui/static/bower_components/moment/src/locale/fr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fr.js rename to ui/static/bower_components/moment/src/locale/fr.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/fy.js b/ui/static/bower_components/moment/src/locale/fy.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/fy.js rename to ui/static/bower_components/moment/src/locale/fy.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/gd.js b/ui/static/bower_components/moment/src/locale/gd.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/gd.js rename to ui/static/bower_components/moment/src/locale/gd.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/gl.js b/ui/static/bower_components/moment/src/locale/gl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/gl.js rename to ui/static/bower_components/moment/src/locale/gl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/he.js b/ui/static/bower_components/moment/src/locale/he.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/he.js rename to ui/static/bower_components/moment/src/locale/he.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/hi.js b/ui/static/bower_components/moment/src/locale/hi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/hi.js rename to ui/static/bower_components/moment/src/locale/hi.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/hr.js b/ui/static/bower_components/moment/src/locale/hr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/hr.js rename to ui/static/bower_components/moment/src/locale/hr.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/hu.js b/ui/static/bower_components/moment/src/locale/hu.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/hu.js rename to ui/static/bower_components/moment/src/locale/hu.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/hy-am.js b/ui/static/bower_components/moment/src/locale/hy-am.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/hy-am.js rename to ui/static/bower_components/moment/src/locale/hy-am.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/id.js b/ui/static/bower_components/moment/src/locale/id.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/id.js rename to ui/static/bower_components/moment/src/locale/id.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/is.js b/ui/static/bower_components/moment/src/locale/is.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/is.js rename to ui/static/bower_components/moment/src/locale/is.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/it.js b/ui/static/bower_components/moment/src/locale/it.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/it.js rename to ui/static/bower_components/moment/src/locale/it.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ja.js b/ui/static/bower_components/moment/src/locale/ja.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ja.js rename to ui/static/bower_components/moment/src/locale/ja.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/jv.js b/ui/static/bower_components/moment/src/locale/jv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/jv.js rename to ui/static/bower_components/moment/src/locale/jv.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ka.js b/ui/static/bower_components/moment/src/locale/ka.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ka.js rename to ui/static/bower_components/moment/src/locale/ka.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/kk.js b/ui/static/bower_components/moment/src/locale/kk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/kk.js rename to ui/static/bower_components/moment/src/locale/kk.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/km.js b/ui/static/bower_components/moment/src/locale/km.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/km.js rename to ui/static/bower_components/moment/src/locale/km.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ko.js b/ui/static/bower_components/moment/src/locale/ko.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ko.js rename to ui/static/bower_components/moment/src/locale/ko.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ky.js b/ui/static/bower_components/moment/src/locale/ky.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ky.js rename to ui/static/bower_components/moment/src/locale/ky.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/lb.js b/ui/static/bower_components/moment/src/locale/lb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/lb.js rename to ui/static/bower_components/moment/src/locale/lb.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/lo.js b/ui/static/bower_components/moment/src/locale/lo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/lo.js rename to ui/static/bower_components/moment/src/locale/lo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/lt.js b/ui/static/bower_components/moment/src/locale/lt.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/lt.js rename to ui/static/bower_components/moment/src/locale/lt.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/lv.js b/ui/static/bower_components/moment/src/locale/lv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/lv.js rename to ui/static/bower_components/moment/src/locale/lv.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/me.js b/ui/static/bower_components/moment/src/locale/me.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/me.js rename to ui/static/bower_components/moment/src/locale/me.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/mi.js b/ui/static/bower_components/moment/src/locale/mi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/mi.js rename to ui/static/bower_components/moment/src/locale/mi.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/mk.js b/ui/static/bower_components/moment/src/locale/mk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/mk.js rename to ui/static/bower_components/moment/src/locale/mk.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ml.js b/ui/static/bower_components/moment/src/locale/ml.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ml.js rename to ui/static/bower_components/moment/src/locale/ml.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/mr.js b/ui/static/bower_components/moment/src/locale/mr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/mr.js rename to ui/static/bower_components/moment/src/locale/mr.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ms-my.js b/ui/static/bower_components/moment/src/locale/ms-my.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ms-my.js rename to ui/static/bower_components/moment/src/locale/ms-my.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ms.js b/ui/static/bower_components/moment/src/locale/ms.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ms.js rename to ui/static/bower_components/moment/src/locale/ms.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/my.js b/ui/static/bower_components/moment/src/locale/my.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/my.js rename to ui/static/bower_components/moment/src/locale/my.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/nb.js b/ui/static/bower_components/moment/src/locale/nb.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/nb.js rename to ui/static/bower_components/moment/src/locale/nb.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ne.js b/ui/static/bower_components/moment/src/locale/ne.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ne.js rename to ui/static/bower_components/moment/src/locale/ne.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/nl-be.js b/ui/static/bower_components/moment/src/locale/nl-be.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/nl-be.js rename to ui/static/bower_components/moment/src/locale/nl-be.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/nl.js b/ui/static/bower_components/moment/src/locale/nl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/nl.js rename to ui/static/bower_components/moment/src/locale/nl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/nn.js b/ui/static/bower_components/moment/src/locale/nn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/nn.js rename to ui/static/bower_components/moment/src/locale/nn.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/pa-in.js b/ui/static/bower_components/moment/src/locale/pa-in.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/pa-in.js rename to ui/static/bower_components/moment/src/locale/pa-in.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/pl.js b/ui/static/bower_components/moment/src/locale/pl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/pl.js rename to ui/static/bower_components/moment/src/locale/pl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/pt-br.js b/ui/static/bower_components/moment/src/locale/pt-br.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/pt-br.js rename to ui/static/bower_components/moment/src/locale/pt-br.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/pt.js b/ui/static/bower_components/moment/src/locale/pt.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/pt.js rename to ui/static/bower_components/moment/src/locale/pt.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ro.js b/ui/static/bower_components/moment/src/locale/ro.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ro.js rename to ui/static/bower_components/moment/src/locale/ro.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ru.js b/ui/static/bower_components/moment/src/locale/ru.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ru.js rename to ui/static/bower_components/moment/src/locale/ru.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/se.js b/ui/static/bower_components/moment/src/locale/se.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/se.js rename to ui/static/bower_components/moment/src/locale/se.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/si.js b/ui/static/bower_components/moment/src/locale/si.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/si.js rename to ui/static/bower_components/moment/src/locale/si.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sk.js b/ui/static/bower_components/moment/src/locale/sk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sk.js rename to ui/static/bower_components/moment/src/locale/sk.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sl.js b/ui/static/bower_components/moment/src/locale/sl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sl.js rename to ui/static/bower_components/moment/src/locale/sl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sq.js b/ui/static/bower_components/moment/src/locale/sq.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sq.js rename to ui/static/bower_components/moment/src/locale/sq.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sr-cyrl.js b/ui/static/bower_components/moment/src/locale/sr-cyrl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sr-cyrl.js rename to ui/static/bower_components/moment/src/locale/sr-cyrl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sr.js b/ui/static/bower_components/moment/src/locale/sr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sr.js rename to ui/static/bower_components/moment/src/locale/sr.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ss.js b/ui/static/bower_components/moment/src/locale/ss.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ss.js rename to ui/static/bower_components/moment/src/locale/ss.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sv.js b/ui/static/bower_components/moment/src/locale/sv.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sv.js rename to ui/static/bower_components/moment/src/locale/sv.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/sw.js b/ui/static/bower_components/moment/src/locale/sw.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/sw.js rename to ui/static/bower_components/moment/src/locale/sw.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/ta.js b/ui/static/bower_components/moment/src/locale/ta.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/ta.js rename to ui/static/bower_components/moment/src/locale/ta.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/te.js b/ui/static/bower_components/moment/src/locale/te.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/te.js rename to ui/static/bower_components/moment/src/locale/te.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tet.js b/ui/static/bower_components/moment/src/locale/tet.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tet.js rename to ui/static/bower_components/moment/src/locale/tet.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/th.js b/ui/static/bower_components/moment/src/locale/th.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/th.js rename to ui/static/bower_components/moment/src/locale/th.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tl-ph.js b/ui/static/bower_components/moment/src/locale/tl-ph.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tl-ph.js rename to ui/static/bower_components/moment/src/locale/tl-ph.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tlh.js b/ui/static/bower_components/moment/src/locale/tlh.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tlh.js rename to ui/static/bower_components/moment/src/locale/tlh.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tr.js b/ui/static/bower_components/moment/src/locale/tr.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tr.js rename to ui/static/bower_components/moment/src/locale/tr.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tzl.js b/ui/static/bower_components/moment/src/locale/tzl.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tzl.js rename to ui/static/bower_components/moment/src/locale/tzl.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tzm-latn.js b/ui/static/bower_components/moment/src/locale/tzm-latn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tzm-latn.js rename to ui/static/bower_components/moment/src/locale/tzm-latn.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/tzm.js b/ui/static/bower_components/moment/src/locale/tzm.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/tzm.js rename to ui/static/bower_components/moment/src/locale/tzm.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/uk.js b/ui/static/bower_components/moment/src/locale/uk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/uk.js rename to ui/static/bower_components/moment/src/locale/uk.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/uz.js b/ui/static/bower_components/moment/src/locale/uz.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/uz.js rename to ui/static/bower_components/moment/src/locale/uz.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/vi.js b/ui/static/bower_components/moment/src/locale/vi.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/vi.js rename to ui/static/bower_components/moment/src/locale/vi.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/x-pseudo.js b/ui/static/bower_components/moment/src/locale/x-pseudo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/x-pseudo.js rename to ui/static/bower_components/moment/src/locale/x-pseudo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/yo.js b/ui/static/bower_components/moment/src/locale/yo.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/yo.js rename to ui/static/bower_components/moment/src/locale/yo.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/zh-cn.js b/ui/static/bower_components/moment/src/locale/zh-cn.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/zh-cn.js rename to ui/static/bower_components/moment/src/locale/zh-cn.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/zh-hk.js b/ui/static/bower_components/moment/src/locale/zh-hk.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/zh-hk.js rename to ui/static/bower_components/moment/src/locale/zh-hk.js diff --git a/themes/bootstrap/public/bower_components/moment/src/locale/zh-tw.js b/ui/static/bower_components/moment/src/locale/zh-tw.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/locale/zh-tw.js rename to ui/static/bower_components/moment/src/locale/zh-tw.js diff --git a/themes/bootstrap/public/bower_components/moment/src/moment.js b/ui/static/bower_components/moment/src/moment.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/src/moment.js rename to ui/static/bower_components/moment/src/moment.js diff --git a/themes/bootstrap/public/bower_components/moment/templates/default.js b/ui/static/bower_components/moment/templates/default.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/templates/default.js rename to ui/static/bower_components/moment/templates/default.js diff --git a/themes/bootstrap/public/bower_components/moment/templates/locale-header.js b/ui/static/bower_components/moment/templates/locale-header.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/templates/locale-header.js rename to ui/static/bower_components/moment/templates/locale-header.js diff --git a/themes/bootstrap/public/bower_components/moment/templates/test-header.js b/ui/static/bower_components/moment/templates/test-header.js similarity index 100% rename from themes/bootstrap/public/bower_components/moment/templates/test-header.js rename to ui/static/bower_components/moment/templates/test-header.js diff --git a/themes/bootstrap/public/favicon.png b/ui/static/favicon.png similarity index 100% rename from themes/bootstrap/public/favicon.png rename to ui/static/favicon.png diff --git a/themes/bootstrap/public/inbucket.css b/ui/static/inbucket.css similarity index 100% rename from themes/bootstrap/public/inbucket.css rename to ui/static/inbucket.css diff --git a/themes/bootstrap/public/mailbox.js b/ui/static/mailbox.js similarity index 100% rename from themes/bootstrap/public/mailbox.js rename to ui/static/mailbox.js diff --git a/themes/bootstrap/public/metrics.js b/ui/static/metrics.js similarity index 100% rename from themes/bootstrap/public/metrics.js rename to ui/static/metrics.js diff --git a/themes/bootstrap/public/monitor.js b/ui/static/monitor.js similarity index 100% rename from themes/bootstrap/public/monitor.js rename to ui/static/monitor.js diff --git a/themes/bootstrap/templates/_base.html b/ui/templates/_base.html similarity index 100% rename from themes/bootstrap/templates/_base.html rename to ui/templates/_base.html diff --git a/themes/bootstrap/templates/mailbox/_html.html b/ui/templates/mailbox/_html.html similarity index 100% rename from themes/bootstrap/templates/mailbox/_html.html rename to ui/templates/mailbox/_html.html diff --git a/themes/bootstrap/templates/mailbox/_show.html b/ui/templates/mailbox/_show.html similarity index 100% rename from themes/bootstrap/templates/mailbox/_show.html rename to ui/templates/mailbox/_show.html diff --git a/themes/bootstrap/templates/mailbox/index.html b/ui/templates/mailbox/index.html similarity index 100% rename from themes/bootstrap/templates/mailbox/index.html rename to ui/templates/mailbox/index.html diff --git a/themes/bootstrap/templates/root/index.html b/ui/templates/root/index.html similarity index 100% rename from themes/bootstrap/templates/root/index.html rename to ui/templates/root/index.html diff --git a/themes/bootstrap/templates/root/monitor.html b/ui/templates/root/monitor.html similarity index 100% rename from themes/bootstrap/templates/root/monitor.html rename to ui/templates/root/monitor.html diff --git a/themes/bootstrap/templates/root/status.html b/ui/templates/root/status.html similarity index 100% rename from themes/bootstrap/templates/root/status.html rename to ui/templates/root/status.html From 04bb842549610c3f77ab87464443af2c99bd0ea3 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 11:55:23 -0700 Subject: [PATCH 41/82] config: Combine TemplateDir and PublicDir into UIDir - Define static names for `templates` and `static` --- pkg/config/config.go | 5 ++--- pkg/rest/testutils_test.go | 3 +-- pkg/server/web/server.go | 12 +++++++++--- pkg/server/web/template.go | 7 +++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2ca6702..5346eea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -57,10 +57,9 @@ type POP3 struct { // Web contains the HTTP server configuration. type Web struct { Addr string `required:"true" default:"0.0.0.0:9000" desc:"Web server IP4 host:port"` - TemplateDir string `required:"true" default:"ui/templates" desc:"Theme template dir"` - TemplateCache bool `required:"true" default:"true" desc:"Cache templates after first use?"` - PublicDir string `required:"true" default:"ui/static" desc:"Theme public dir"` + UIDir string `required:"true" default:"ui" desc:"User interface dir"` GreetingFile string `required:"true" default:"ui/greeting.html" desc:"Home page greeting HTML"` + TemplateCache bool `required:"true" default:"true" desc:"Cache templates after first use?"` MailboxPrompt string `required:"true" default:"@inbucket" desc:"Prompt next to mailbox input"` CookieAuthKey string `desc:"Session cipher key (text)"` MonitorVisible bool `required:"true" default:"true" desc:"Show monitor tab in UI?"` diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index e2f2e3d..b062083 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -36,8 +36,7 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { http.DefaultServeMux = http.NewServeMux() cfg := &config.Root{ Web: config.Web{ - TemplateDir: "../themes/bootstrap/templates", - PublicDir: "../themes/bootstrap/public", + UIDir: "../ui", }, } shutdownChan := make(chan bool) diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index 96716f0..8e5524c 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -6,6 +6,7 @@ import ( "expvar" "net" "net/http" + "path/filepath" "time" "github.com/gorilla/mux" @@ -20,6 +21,11 @@ import ( // Handler is a function type that handles an HTTP request in Inbucket type Handler func(http.ResponseWriter, *http.Request, *Context) error +const ( + staticDir = "static" + templateDir = "templates" +) + var ( // msgHub holds a reference to the message pub/sub system msgHub *msghub.Hub @@ -59,10 +65,10 @@ func Initialize( manager = mm // Content Paths - log.Infof("HTTP templates mapped to %q", conf.Web.TemplateDir) - log.Infof("HTTP static content mapped to %q", conf.Web.PublicDir) + staticPath := filepath.Join(conf.Web.UIDir, staticDir) + log.Infof("Web UI content mapped to path: %s", conf.Web.UIDir) Router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", - http.FileServer(http.Dir(conf.Web.PublicDir)))) + http.FileServer(http.Dir(staticPath)))) http.Handle("/", Router) // Session cookie setup diff --git a/pkg/server/web/template.go b/pkg/server/web/template.go index 2597ada..87675a5 100644 --- a/pkg/server/web/template.go +++ b/pkg/server/web/template.go @@ -5,7 +5,6 @@ import ( "net/http" "path" "path/filepath" - "strings" "sync" "github.com/jhillyerd/inbucket/pkg/log" @@ -49,8 +48,7 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { return t, nil } - tempPath := strings.Replace(name, "/", string(filepath.Separator), -1) - tempFile := filepath.Join(rootConfig.Web.TemplateDir, tempPath) + tempFile := filepath.Join(rootConfig.Web.UIDir, templateDir, filepath.FromSlash(name)) log.Tracef("Parsing template %v", tempFile) var err error @@ -62,7 +60,8 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { t, err = t.ParseFiles(tempFile) } else { t = template.New("_base.html").Funcs(TemplateFuncs) - t, err = t.ParseFiles(filepath.Join(rootConfig.Web.TemplateDir, "_base.html"), tempFile) + t, err = t.ParseFiles( + filepath.Join(rootConfig.Web.UIDir, templateDir, "_base.html"), tempFile) } if err != nil { return nil, err From 69a0d355f99a817f3fb66f84af424d700ab9b617 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 10:06:03 -0700 Subject: [PATCH 42/82] doc: Add doc/config.md to document config for #86 - Increase default max message size to 10MB. --- doc/config.md | 349 +++++++++++++++++++++++++++++++++++++ pkg/config/config.go | 2 +- pkg/storage/mem/maxsize.go | 4 +- 3 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 doc/config.md diff --git a/doc/config.md b/doc/config.md new file mode 100644 index 0000000..5f9fa92 --- /dev/null +++ b/doc/config.md @@ -0,0 +1,349 @@ +# Inbucket Configuration + +Inbucket is configured via environment variables. Most options have a +reasonable default, but it is likely you will need to change some to suite your +desired use cases. + +Running `inbucket -help` will yield a condensed summary of the environment +variables it supports: + + KEY DEFAULT DESCRIPTION + INBUCKET_LOGLEVEL INFO TRACE, INFO, WARN, or ERROR + INBUCKET_SMTP_ADDR 0.0.0.0:2500 SMTP server IP4 host:port + INBUCKET_SMTP_DOMAIN inbucket HELO domain + INBUCKET_SMTP_DOMAINNOSTORE Load testing domain + INBUCKET_SMTP_MAXRECIPIENTS 200 Maximum RCPT TO per message + INBUCKET_SMTP_MAXMESSAGEBYTES 10240000 Maximum message size + INBUCKET_SMTP_STOREMESSAGES true Store incoming mail? + INBUCKET_SMTP_TIMEOUT 300s Idle network timeout + INBUCKET_POP3_ADDR 0.0.0.0:1100 POP3 server IP4 host:port + INBUCKET_POP3_DOMAIN inbucket HELLO domain + INBUCKET_POP3_TIMEOUT 600s Idle network timeout + INBUCKET_WEB_ADDR 0.0.0.0:9000 Web server IP4 host:port + INBUCKET_WEB_UIDIR ui User interface dir + INBUCKET_WEB_GREETINGFILE ui/greeting.html Home page greeting HTML + INBUCKET_WEB_TEMPLATECACHE true Cache templates after first use? + INBUCKET_WEB_MAILBOXPROMPT @inbucket Prompt next to mailbox input + INBUCKET_WEB_COOKIEAUTHKEY Session cipher key (text) + INBUCKET_WEB_MONITORVISIBLE true Show monitor tab in UI? + INBUCKET_WEB_MONITORHISTORY 30 Monitor remembered messages + INBUCKET_STORAGE_TYPE memory Storage impl: file or memory + INBUCKET_STORAGE_PARAMS Storage impl parameters, see docs. + INBUCKET_STORAGE_RETENTIONPERIOD 24h Duration to retain messages + INBUCKET_STORAGE_RETENTIONSLEEP 50ms Duration to sleep between mailboxes + INBUCKET_STORAGE_MAILBOXMSGCAP 500 Maximum messages per mailbox + +The following documentation will describe each of these in more detail. + + +## Global + +### Log Level + +`INBUCKET_LOGLEVEL` + +This setting controls the verbosity of log output. A small desktop installation +should probably select INFO, but a busy shared installation would be better off +with WARN or ERROR. + +- Default: `INFO` +- Values: one of `TRACE`, `INFO`, `WARN`, or `ERROR` + + +## SMTP + +### Address and Port + +`INBUCKET_SMTP_ADDR` + +The IPv4 address and TCP port number the SMTP server should listen on, separated +by a colon. Some operating systems may prevent Inbucket from listening on port +25 without escalated privileges. Using an IP address of 0.0.0.0 will cause +Inbucket to listen on all available network interfaces. + +- Default: `0.0.0.0:2500` + +### Greeting Domain + +`INBUCKET_SMTP_DOMAIN` + +The domain used in the SMTP greeting: + + 220 domain Inbucket SMTP ready + +Most SMTP clients appear to ignore this value. + +- Default: `inbucket` + +### Load Testing/No Store Domain + +`INBUCKET_SMTP_DOMAINNOSTORE` + +Mail sent to this domain will not be stored by Inbucket. This is helpful if you +are load or soak testing a service, and do not plan to inspect the resulting +emails. Messages sent to a domain other than this will be stored normally. + +- Default: None +- Example: `bitbucket.local` + +### Maximum Recipients + +`INBUCKET_SMTP_MAXRECIPIENTS` + +Maximum number of recipients allowed (SMTP `RCPT TO` phase). If you are testing +a mailing list server, you may need to increase this value. For comparison, the +Postfix SMTP server uses a default of 1000, it would be unwise to exceed this. + +- Default: `200` + +### Maximum Message Size + +`INBUCKET_SMTP_MAXMESSAGEBYTES` + +Maximum allowable size of a message (including headers) in bytes. Messages +exceeding this size will be rejected during the SMTP `DATA` phase. + +- Default: `10240000` (10MB) + +### Store Messages + +`INBUCKET_SMTP_STOREMESSAGES` + +This option can be used to disable mail storage entirely. Useful for load +testing, or turning Inbucket into a black hole that will consume our entire +solar system. + +- Default: `true` +- Values: `true` or `false` + +### Network Idle Timeout + +`INBUCKET_SMTP_TIMEOUT` + +Delay before closing an idle SMTP connection. The SMTP RFC recommends 300 +seconds. Consider reducing this *significantly* if you plan to expose Inbucket +to the public internet. + +- Default: `300s` +- Values: Duration ending in `s` for seconds, `m` for minutes + + +## POP3 + +### Address and Port + +`INBUCKET_POP3_ADDR` + +The IPv4 address and TCP port number the POP3 server should listen on, separated +by a colon. Some operating systems may prevent Inbucket from listening on port +110 without escalated privileges. Using an IP address of 0.0.0.0 will cause +Inbucket to listen on all available network interfaces. + +- Default: `0.0.0.0:1100` + +### Greeting Domain + +`INBUCKET_POP3_DOMAIN` + +The domain used in the POP3 greeting: + + +OK Inbucket POP3 server ready <26641.1522000423@domain> + +Most POP3 clients appear to ignore this value. + +- Default: `inbucket` + +### Network Idle Timeout + +`INBUCKET_POP3_TIMEOUT` + +Delay before closing an idle POP3 connection. The POP3 RFC recommends 600 +seconds. Consider reducing this *significantly* if you plan to expose Inbucket +to the public internet. + +- Default: `600s` +- Values: Duration ending in `s` for seconds, `m` for minutes + + +## Web + +### Address and Port + +`INBUCKET_WEB_ADDR` + +The IPv4 address and TCP port number the HTTP server should listen on, separated +by a colon. Some operating systems may prevent Inbucket from listening on port +80 without escalated privileges. Using an IP address of 0.0.0.0 will cause +Inbucket to listen on all available network interfaces. + +- Default: `0.0.0.0:9000` + +### UI Directory + +`INBUCKET_WEB_UIDIR` + +This directory contains the templates and static assets for the web user +interface. You will need to change this if the current working directory +doesn't contain the `ui` directory at startup. + +Inbucket will load templates from the `templates` sub-directory, and serve +static assets from the `static` sub-directory. + +- Default: `ui` +- Values: Operating system specific path syntax + +### Greeting HTML File + +`INBUCKET_WEB_GREETINGFILE` + +The content of the greeting file will be injected into the front page of +Inbucket. It can be used to instruct users on how to send mail into your +Inbucket installation, as well as link to REST documentation, etc. + +- Default: `ui/greeting.html` + +### Template Caching + +`INBUCKET_WEB_TEMPLATECACHE` + +Tells Inbucket to cache parsed template files. This should be left as default +unless you are a developer working on the Inbucket web interface. + +- Default: `true` +- Values: `true` or `false` + +### Mailbox Prompt + +`INBUCKET_WEB_MAILBOXPROMPT` + +Text prompt displayed to the right of the mailbox name input field in the web +interface. Can be used to nudge your users into typing just the mailbox name +instead of an entire email address. + +Set to an empty string to hide the prompt. + +- Default: `@inbucket` + +### Cookie Authentication Key + +`INBUCKET_WEB_COOKIEAUTHKEY` + +Inbucket stores session information in an encrypted browser cookie. Unless +specified, Inbucket generates a random key at startup. The only notable data +stored in a user session is the list of recently accessed mailboxes. + +- Default: None +- Value: Text string, no particular format required + +### Monitor Visible + +`INBUCKET_WEB_MONITORVISIBLE` + +If true, the Monitor tab will be available, allowing users to observe all +messages received by Inbucket as they arrive. Disabling the monitor facilitates +security through obscurity. + +This setting has no impact on the availability of the underlying WebSocket, +which may be used by other parts of the Inbucket interface or continuous +integration tests. + +- Default: `true` +- Values: `true` or `false` + +### Monitor History + +`INBUCKET_WEB_MONITORHISTORY` + +The number of messages to remember on the *server* for new Monitor clients. +Does not impact the amount of *new* messages displayed by the Monitor. +Increasing this has no appreciable impact on memory use, but may slow down the +Monitor user interface. + +This setting has the same effect on the amount of messages available via +WebSocket. + +Setting to 0 will disable the monitor, but will probably break new mail +notifications in the web interface when I finally get around to implementing +them. + +- Default: `30` +- Values: Integer greater than or equal to 0 + + +## Storage + +### Type + +`INBUCKET_STORAGE_TYPE` + +Selects the storage implementation to use. Currently Inbucket supports two: + +- `file`: stores messages as individual files in a nested directory structure + based on the hash of the mailbox name. Each mailbox also includes an index + file to speed up enumeration of the mailbox contents. +- `memory`: stores messages in RAM, they will be lost if Inbucket is restarted, + or crashes, etc. + +File storage is recommended for larger/shared installations. Memory is better +suited to desktop or continuous integration test use cases. + +- Default: `memory` +- Values: `file` or `memory` + +### Parameters + +`INBUCKET_STORAGE_PARAMS` + +Parameters specific to the storage type selected. Formatted as a comma +separated list of key:value pairs. + +- Default: None +- Examples: `maxkb=10240` or `path=/tmp/inbucket` + +#### file parameters + +- `path`: Operating system specific path to the directory where mail should be + stored. + +#### memory parameters + +- `maxkb`: Maximum size of the mail store in kilobytes. The oldest messages in + the store will be deleted to enforce the limit. In-memory storage has some + overhead, for now it is recommended to set this to half the total amount of + memory you are willing to allocate to Inbucket. + +### Retention Period + +`INBUCKET_STORAGE_RETENTIONPERIOD` + +If set, Inbucket will scan the contents of its mail store once per minute, +removing messages older than this. This will be enforced regardless of the type +of storage configured. + +- Default: `24h` +- Values: Duration ending in `m` for minutes, `h` for hours. Should be + significantly longer than one minute, or `0` to disable. + +### Retention Sleep + +`INBUCKET_STORAGE_RETENTIONSLEEP` + +Duration to sleep between scanning each mailbox for expired messages. +Increasing this number will reduce disk thrashing, but extend the length of time +required to complete a scan of the entire mail store. + +This delay is still enforced for memory stores, but could be reduced from the +default. Setting to `0` may degrade performance of HTTP/SMTP/POP3 services. + +- Default: `50ms` +- Values: Duration ending in `ms` for milliseconds, `s` for seconds + +### Per Mailbox Message Cap + +`INBUCKET_STORAGE_MAILBOXMSGCAP` + +Maximum messages allowed in a single mailbox, exceeding this will cause older +messages to be deleted from the mailbox. + +- Default: `500` +- Values: Positive integer, or `0` to disable diff --git a/pkg/config/config.go b/pkg/config/config.go index 5346eea..dd35f39 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -42,7 +42,7 @@ type SMTP struct { Domain string `required:"true" default:"inbucket" desc:"HELO domain"` DomainNoStore string `desc:"Load testing domain"` MaxRecipients int `required:"true" default:"200" desc:"Maximum RCPT TO per message"` - MaxMessageBytes int `required:"true" default:"2048000" desc:"Maximum message size"` + MaxMessageBytes int `required:"true" default:"10240000" desc:"Maximum message size"` StoreMessages bool `required:"true" default:"true" desc:"Store incoming mail?"` Timeout time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` } diff --git a/pkg/storage/mem/maxsize.go b/pkg/storage/mem/maxsize.go index 888247b..3b11850 100644 --- a/pkg/storage/mem/maxsize.go +++ b/pkg/storage/mem/maxsize.go @@ -7,8 +7,8 @@ type msgDone struct { done chan struct{} } -// enforceMaxSize will delete the oldest message until the entire mail store is equal to or less -// than Store.maxSize bytes. +// maxSizeEnforcer will delete the oldest message until the entire mail store is equal to or less +// than maxSize bytes. func (s *Store) maxSizeEnforcer(maxSize int64) { all := &list.List{} curSize := int64(0) From ce2339ee9ce81f9cfcd4533e50598b780346c46e Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 14:30:34 -0700 Subject: [PATCH 43/82] conf: Delete obsolete config files for #86 --- etc/devel.conf | 129 ----------------------------- etc/docker/defaults/inbucket.conf | 131 ------------------------------ etc/homebrew/inbucket.conf | 131 ------------------------------ etc/inbucket.conf | 129 ----------------------------- etc/unix-sample.conf | 129 ----------------------------- etc/win-sample.conf | 129 ----------------------------- 6 files changed, 778 deletions(-) delete mode 100644 etc/devel.conf delete mode 100644 etc/docker/defaults/inbucket.conf delete mode 100644 etc/homebrew/inbucket.conf delete mode 100644 etc/inbucket.conf delete mode 100644 etc/unix-sample.conf delete mode 100644 etc/win-sample.conf diff --git a/etc/devel.conf b/etc/devel.conf deleted file mode 100644 index 1e765d4..0000000 --- a/etc/devel.conf +++ /dev/null @@ -1,129 +0,0 @@ -# devel.conf -# Sample development configuration - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -install.dir=. -default.domain=inbucket.local - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=TRACE - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=2500 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=30 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=20480000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=1100 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=9000 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(install.dir)s/themes/%(theme)s/templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=false - -# Path to the selected themes public (static) files -public.dir=%(install.dir)s/themes/%(theme)s/public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=%(install.dir)s/themes/greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=/tmp/inbucket - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=0 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=100 diff --git a/etc/docker/defaults/inbucket.conf b/etc/docker/defaults/inbucket.conf deleted file mode 100644 index 8825e87..0000000 --- a/etc/docker/defaults/inbucket.conf +++ /dev/null @@ -1,131 +0,0 @@ -# inbucket.conf -# Configuration for Inbucket inside of Docker -# -# These should be reasonable defaults for a production install of Inbucket - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -install.dir=/opt/inbucket -default.domain=inbucket.local - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=INFO - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=10025 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=300 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=2048000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=10110 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=10080 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(install.dir)s/themes/%(theme)s/templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=true - -# Path to the selected themes public (static) files -public.dir=%(install.dir)s/themes/%(theme)s/public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=/con/configuration/greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -#cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=/con/data - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=4320 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=300 diff --git a/etc/homebrew/inbucket.conf b/etc/homebrew/inbucket.conf deleted file mode 100644 index bebd42a..0000000 --- a/etc/homebrew/inbucket.conf +++ /dev/null @@ -1,131 +0,0 @@ -# inbucket.conf -# homebrew inbucket configuration -# {{}} values will be replaced during installation - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -default.domain=inbucket.local -themes.dir={{HOMEBREW_PREFIX}}/share/inbucket/themes -datastore.dir={{HOMEBREW_PREFIX}}/var/inbucket/datastore - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=INFO - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=2500 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=300 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=2048000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=1100 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=9000 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(themes.dir)s/%(theme)s/templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=true - -# Path to the selected themes public (static) files -public.dir=%(themes.dir)s/%(theme)s/public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=%(themes.dir)s/greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=%(datastore.dir)s - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=10080 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=100 diff --git a/etc/inbucket.conf b/etc/inbucket.conf deleted file mode 100644 index e8e16d4..0000000 --- a/etc/inbucket.conf +++ /dev/null @@ -1,129 +0,0 @@ -# inbucket.conf -# Sample inbucket configuration - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -install.dir=. -default.domain=inbucket.local - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=INFO - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=2500 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -#domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=300 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=2048000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=1100 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=9000 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(install.dir)s/themes/%(theme)s/templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=true - -# Path to the selected themes public (static) files -public.dir=%(install.dir)s/themes/%(theme)s/public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=%(install.dir)s/themes/greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -#cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=/tmp/inbucket - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=240 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=500 diff --git a/etc/unix-sample.conf b/etc/unix-sample.conf deleted file mode 100644 index 240923f..0000000 --- a/etc/unix-sample.conf +++ /dev/null @@ -1,129 +0,0 @@ -# inbucket.conf -# Sample inbucket configuration - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -install.dir=/opt/inbucket -default.domain=inbucket.local - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=INFO - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=25 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -#domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=300 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=2048000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=110 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=80 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(install.dir)s/themes/%(theme)s/templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=true - -# Path to the selected themes public (static) files -public.dir=%(install.dir)s/themes/%(theme)s/public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=%(install.dir)s/themes/greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -#cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=/var/opt/inbucket - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=240 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=500 diff --git a/etc/win-sample.conf b/etc/win-sample.conf deleted file mode 100644 index 398dbd0..0000000 --- a/etc/win-sample.conf +++ /dev/null @@ -1,129 +0,0 @@ -# win-sample.conf -# Sample inbucket configuration for Windows - -############################################################################# -[DEFAULT] - -# Not used directly, but is typically referenced below in %()s format. -install.dir=. -default.domain=inbucket.local - -############################################################################# -[logging] - -# Options from least to most verbose: ERROR, WARN, INFO, TRACE -level=INFO - -############################################################################# -[smtp] - -# IPv4 address to listen for SMTP connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for SMTP connections on. -ip4.port=2500 - -# used in SMTP greeting -domain=%(default.domain)s - -# optional: mail sent to accounts at this domain will not be stored, -# for mixed use (content and load testing) -#domain.nostore=bitbucket.local - -# Maximum number of RCPT TO: addresses we allow from clients, the SMTP -# RFC recommends this be at least 100. -max.recipients=100 - -# How long we allow a network connection to be idle before hanging up on the -# client, SMTP RFC recommends at least 5 minutes (300 seconds). -max.idle.seconds=300 - -# Maximum allowable size of message body in bytes (including attachments) -max.message.bytes=2048000 - -# Should we place messages into the datastore, or just throw them away -# (for load testing): true or false -store.messages=true - -############################################################################# -[pop3] - -# IPv4 address to listen for POP3 connections on. -ip4.address=0.0.0.0 - -# IPv4 port to listen for POP3 connections on. -ip4.port=1100 - -# used in POP3 greeting -domain=%(default.domain)s - -# How long we allow a network connection to be idle before hanging up on the -# client, POP3 RFC requires at least 10 minutes (600 seconds). -max.idle.seconds=600 - -############################################################################# -[web] - -# IPv4 address to serve HTTP web interface on -ip4.address=0.0.0.0 - -# IPv4 port to serve HTTP web interface on -ip4.port=9000 - -# Name of web theme to use -theme=bootstrap - -# Prompt displayed between the mailbox entry field and View button. Leave -# empty or comment out to hide the prompt. -mailbox.prompt=@inbucket - -# Path to the selected themes template files -template.dir=%(install.dir)s\themes\%(theme)s\templates - -# Should we cache parsed templates (set to false during theme dev) -template.cache=true - -# Path to the selected themes public (static) files -public.dir=%(install.dir)s\themes\%(theme)s\public - -# Path to the greeting HTML displayed on front page, can be moved out of -# installation dir for customization -greeting.file=%(install.dir)s\themes\greeting.html - -# Key used to sign session cookie data so that it cannot be tampered with. -# If this is left unset, Inbucket will generate a random key at startup -# and previous sessions will be invalidated. -#cookie.auth.key=secret-inbucket-session-cookie-key - -# Enable or disable the live message monitor tab for the web UI. This will let -# anybody see all messages delivered to Inbucket. This setting has no impact -# on the availability of the underlying WebSocket. -monitor.visible=true - -# How many historical message headers should be cached for display by new -# monitor connections. It does not limit the number of messages displayed by -# the browser once the monitor is open; all freshly received messages will be -# appended to the on screen list. This setting also affects the underlying -# API/WebSocket. -monitor.history=30 - -############################################################################# -[datastore] - -# Path to the datastore, mail will be written into subdirectories -path=.\inbucket-data - -# How many minutes after receipt should a message be stored until it's -# automatically purged. To retain messages until manually deleted, set this -# to 0 -retention.minutes=240 - -# How many milliseconds to sleep after purging messages from a mailbox. -# This should help reduce disk I/O when there are a large number of messages -# to purge. -retention.sleep.millis=100 - -# Maximum number of messages we will store in a single mailbox. If this -# number is exceeded, the oldest message in the box will be deleted each -# time a new message is received for it. -mailbox.message.cap=500 From 86c8ccf9eaf10bae3989611476bba785440e1dc5 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 15:35:45 -0700 Subject: [PATCH 44/82] docker: Update for environment config for #86 - Change to default ports (less surprising) - Drop `/con/` volume naming, never caught on --- CHANGELOG.md | 4 ++++ Dockerfile | 29 +++++++++++++++++---------- etc/docker/defaults/start-inbucket.sh | 3 +-- etc/docker/install.sh | 5 ++--- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aba3b7..7e933c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). more enjoyable to work on. - Renamed `themes` directory to `ui` and eliminated the intermediate `bootstrap` directory. +- Docker build: + - Uses the same default ports as other builds; smtp:2500 http:9000 pop3:1100 + - Uses volume `/config` for `greeting.html` + - Uses volume `/storage` for mail storage ## [v1.3.1] - 2018-03-10 diff --git a/Dockerfile b/Dockerfile index ba393f5..5bc5da5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,23 +2,30 @@ # Inbucket website: http://www.inbucket.org/ FROM golang:1.10-alpine -MAINTAINER James Hillyerd, @jameshillyerd -# Configuration (WORKDIR doesn't support env vars) +# Configuration ENV INBUCKET_SRC $GOPATH/src/github.com/jhillyerd/inbucket ENV INBUCKET_HOME /opt/inbucket -WORKDIR $INBUCKET_HOME -ENTRYPOINT ["/con/context/start-inbucket.sh"] -CMD ["/con/configuration/inbucket.conf"] +ENV INBUCKET_SMTP_DOMAINNOSTORE bitbucket.local +ENV INBUCKET_SMTP_TIMEOUT 30s +ENV INBUCKET_POP3_TIMEOUT 30s +ENV INBUCKET_WEB_UIDIR $INBUCKET_HOME/ui +ENV INBUCKET_WEB_GREETINGFILE /config/greeting.html +ENV INBUCKET_WEB_COOKIEAUTHKEY secret-inbucket-session-cookie-key +ENV INBUCKET_STORAGE_TYPE file +ENV INBUCKET_STORAGE_PARAMS path:/storage +ENV INBUCKET_STORAGE_RETENTIONPERIOD 72h +ENV INBUCKET_STORAGE_MAILBOXMSGCAP 300 # Ports: SMTP, HTTP, POP3 -EXPOSE 10025 10080 10110 +EXPOSE 2500 9000 1100 -# Persistent Volumes, following convention at: -# https://github.com/docker/docker/issues/9277 -# NOTE /con/context is also used, not exposed by default -VOLUME /con/configuration -VOLUME /con/data +# Persistent Volumes +VOLUME /config +VOLUME /storage + +WORKDIR $INBUCKET_HOME +ENTRYPOINT "/start-inbucket.sh" # Build Inbucket COPY . $INBUCKET_SRC/ diff --git a/etc/docker/defaults/start-inbucket.sh b/etc/docker/defaults/start-inbucket.sh index 3f30e6b..7315ee7 100755 --- a/etc/docker/defaults/start-inbucket.sh +++ b/etc/docker/defaults/start-inbucket.sh @@ -3,7 +3,7 @@ # description: start inbucket (runs within a docker container) CONF_SOURCE="$INBUCKET_HOME/defaults" -CONF_TARGET="/con/configuration" +CONF_TARGET="/config" set -eo pipefail @@ -18,7 +18,6 @@ install_default_config() { fi } -install_default_config "inbucket.conf" install_default_config "greeting.html" exec "$INBUCKET_HOME/bin/inbucket" $* diff --git a/etc/docker/install.sh b/etc/docker/install.sh index 54507db..413f8de 100755 --- a/etc/docker/install.sh +++ b/etc/docker/install.sh @@ -35,10 +35,9 @@ set -x mkdir -p "$bindir" install inbucket "$bindir" mkdir -p "$contextdir" -install etc/docker/defaults/start-inbucket.sh "$contextdir" -cp -r themes "$installdir/" +install etc/docker/defaults/start-inbucket.sh / +cp -r ui "$installdir/" mkdir -p "$defaultsdir" -cp etc/docker/defaults/inbucket.conf "$defaultsdir" cp etc/docker/defaults/greeting.html "$defaultsdir" set +x From 2d09e94f873e15da14a6938cff28dab959751de3 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 16:08:34 -0700 Subject: [PATCH 45/82] log: Fix another deadlock. --- pkg/log/stdout_unix.go | 11 ++++++----- pkg/log/stdout_windows.go | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/log/stdout_unix.go b/pkg/log/stdout_unix.go index 1197844..0908102 100644 --- a/pkg/log/stdout_unix.go +++ b/pkg/log/stdout_unix.go @@ -3,8 +3,10 @@ package log import ( - "golang.org/x/sys/unix" + "log" "os" + + "golang.org/x/sys/unix" ) // closeStdin will close stdin on Unix platforms - this is standard practice @@ -12,20 +14,19 @@ import ( func closeStdin() { if err := os.Stdin.Close(); err != nil { // Not a fatal error - Errorf("Failed to close os.Stdin during log setup") + log.Printf("Failed to close os.Stdin during log setup") } } // reassignStdout points stdout/stderr to our logfile on systems that support // the Dup2 syscall per https://github.com/golang/go/issues/325 func reassignStdout() { - Tracef("Unix reassignStdout()") if err := unix.Dup2(int(logf.Fd()), 1); err != nil { // Not considered fatal - Errorf("Failed to re-assign stdout to logfile: %v", err) + log.Printf("Failed to re-assign stdout to logfile: %v", err) } if err := unix.Dup2(int(logf.Fd()), 2); err != nil { // Not considered fatal - Errorf("Failed to re-assign stderr to logfile: %v", err) + log.Printf("Failed to re-assign stderr to logfile: %v", err) } } diff --git a/pkg/log/stdout_windows.go b/pkg/log/stdout_windows.go index ad8d829..f883d24 100644 --- a/pkg/log/stdout_windows.go +++ b/pkg/log/stdout_windows.go @@ -3,6 +3,7 @@ package log import ( + "log" "os" ) @@ -16,7 +17,6 @@ func closeStdin() { // reassignStdout points stdout/stderr to our logfile on systems that do not // support the Dup2 syscall func reassignStdout() { - Tracef("Windows reassignStdout()") if !stdOutsClosed { // Close std* streams to prevent accidental output, they will be redirected to // our logfile below @@ -24,11 +24,11 @@ func reassignStdout() { // Warning: this will hide panic() output, sorry Windows users if err := os.Stderr.Close(); err != nil { // Not considered fatal - Errorf("Failed to close os.Stderr during log setup") + log.Printf("Failed to close os.Stderr during log setup") } if err := os.Stdin.Close(); err != nil { // Not considered fatal - Errorf("Failed to close os.Stdin during log setup") + log.Printf("Failed to close os.Stdin during log setup") } os.Stdout = logf os.Stderr = logf From 23dc3572020fc0422ae55c8e2285986480aeb793 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 16:16:06 -0700 Subject: [PATCH 46/82] etc: Add dev-start.sh script for #86 --- etc/dev-start.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 etc/dev-start.sh diff --git a/etc/dev-start.sh b/etc/dev-start.sh new file mode 100755 index 0000000..6112cf3 --- /dev/null +++ b/etc/dev-start.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# dev-start.sh +# description: Developer friendly Inbucket configuration + +export INBUCKET_LOGLEVEL="TRACE" +export INBUCKET_SMTP_DOMAINNOSTORE="bitbucket.local" +export INBUCKET_WEB_TEMPLATECACHE="false" +export INBUCKET_WEB_COOKIEAUTHKEY="not-secret" +export INBUCKET_STORAGE_TYPE="file" +export INBUCKET_STORAGE_PARAMS="path:/tmp/inbucket" +export INBUCKET_STORAGE_RETENTIONPERIOD="5m" + +if ! test -x ./inbucket; then + echo "$PWD/inbucket not found/executable!" >&2 + echo "Run this script from the inbucket root directory after running make" >&2 + exit 1 +fi + +exec ./inbucket $* From 06989c8218ca8af4865db7e93e26438cb136eb43 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 16:28:37 -0700 Subject: [PATCH 47/82] Update goreleaser config for #86 - Remove inbucket.bat, with new env defaults Windows does not need a script to launch. --- .goreleaser.yml | 5 ++--- inbucket.bat | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 inbucket.bat diff --git a/.goreleaser.yml b/.goreleaser.yml index f8fe52c..d4c588b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -20,7 +20,7 @@ builds: - amd64 goarm: - "6" - main: . + main: ./cmd/inbucket ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} - binary: client goos: @@ -46,9 +46,8 @@ archive: - LICENSE* - README* - CHANGELOG* - - inbucket.bat - etc/**/* - - themes/**/* + - ui/**/* fpm: bindir: /usr/local/bin snapshot: diff --git a/inbucket.bat b/inbucket.bat deleted file mode 100644 index 0737139..0000000 --- a/inbucket.bat +++ /dev/null @@ -1 +0,0 @@ -inbucket.exe etc\win-sample.conf \ No newline at end of file From 0055b8491686acf8b876b1b445508264ed70b8f2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 18:23:25 -0700 Subject: [PATCH 48/82] debian: Use goreleaser to generate .deb package for #89 --- .goreleaser.yml | 26 ++++++++++++++++++++++--- CHANGELOG.md | 1 + etc/linux/inbucket.service | 33 ++++++++++++++++++++++++++++++++ etc/ubuntu/README | 3 --- etc/ubuntu/inbucket-upstart.conf | 29 ---------------------------- etc/ubuntu/inbucket.logrotate | 8 -------- etc/ubuntu/inbucket.service | 20 ------------------- 7 files changed, 57 insertions(+), 63 deletions(-) create mode 100644 etc/linux/inbucket.service delete mode 100644 etc/ubuntu/README delete mode 100644 etc/ubuntu/inbucket-upstart.conf delete mode 100644 etc/ubuntu/inbucket.logrotate delete mode 100644 etc/ubuntu/inbucket.service diff --git a/.goreleaser.yml b/.goreleaser.yml index d4c588b..864d60f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,14 +1,17 @@ project_name: inbucket + release: github: owner: jhillyerd name: inbucket name_template: '{{.Tag}}' + brew: commit_author: name: goreleaserbot email: goreleaser@carlosbecker.com install: bin.install "" + builds: - binary: inbucket goos: @@ -22,7 +25,7 @@ builds: - "6" main: ./cmd/inbucket ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} - - binary: client + - binary: inbucket-client goos: - darwin - freebsd @@ -34,6 +37,7 @@ builds: - "6" main: ./cmd/client ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} + archive: format: tar.gz wrap_in_directory: true @@ -48,12 +52,28 @@ archive: - CHANGELOG* - etc/**/* - ui/**/* -fpm: - bindir: /usr/local/bin + +nfpm: + vendor: inbucket.org + homepage: https://www.inbucket.org/ + maintainer: github@hillyerd.com + description: All-in-one disposable webmail service. + license: MIT + formats: + - deb + files: + "ui/**/*": "/usr/local/share/inbucket/ui" + config_files: + "etc/linux/inbucket.service": "/lib/systemd/system/inbucket.service" + "ui/greeting.html": "/etc/inbucket/greeting.html" + snapshot: name_template: SNAPSHOT-{{ .Commit }} + checksum: name_template: '{{ .ProjectName }}_{{ .Version }}_checksums.txt' + dist: dist + sign: artifacts: none diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e933c5..fd77963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Storage type is now displayed on Status page. - Store size is now calculated during retention scan and displayed on the Status page. +- Debian `.deb` package generation to release process. ### Changed - Massive refactor of back-end code. Inbucket should now be both easier and diff --git a/etc/linux/inbucket.service b/etc/linux/inbucket.service new file mode 100644 index 0000000..00ed9dc --- /dev/null +++ b/etc/linux/inbucket.service @@ -0,0 +1,33 @@ +[Unit] +Description=Inbucket Disposable Email Service +After=network.target + +[Service] +Type=simple +User=daemon +Group=daemon +PermissionsStartOnly=true + +Environment=INBUCKET_LOGLEVEL=WARN +Environment=INBUCKET_SMTP_ADDR=0.0.0.0:2500 +Environment=INBUCKET_POP3_ADDR=0.0.0.0:1100 +Environment=INBUCKET_WEB_ADDR=0.0.0.0:9000 +Environment=INBUCKET_WEB_UIDIR=/usr/local/share/inbucket/ui +Environment=INBUCKET_WEB_GREETINGFILE=/etc/inbucket/greeting.html +Environment=INBUCKET_STORAGE_TYPE=file +Environment=INBUCKET_STORAGE_PARAMS=path:/var/local/inbucket + +# Uncomment line below to use low numbered ports +#ExecStartPre=/sbin/setcap 'cap_net_bind_service=+ep' /usr/local/bin/inbucket + +ExecStartPre=/bin/mkdir -p /var/local/inbucket +ExecStartPre=/bin/chown daemon:daemon /var/local/inbucket + +ExecStart=/usr/local/bin/inbucket + +# Give SMTP connections time to drain +TimeoutStopSec=20 +KillMode=mixed + +[Install] +WantedBy=multi-user.target diff --git a/etc/ubuntu/README b/etc/ubuntu/README deleted file mode 100644 index f2559bd..0000000 --- a/etc/ubuntu/README +++ /dev/null @@ -1,3 +0,0 @@ -Please see the Ubuntu installation guide on our website: - -http://www.inbucket.org/installation/ubuntu.html diff --git a/etc/ubuntu/inbucket-upstart.conf b/etc/ubuntu/inbucket-upstart.conf deleted file mode 100644 index 810b6a9..0000000 --- a/etc/ubuntu/inbucket-upstart.conf +++ /dev/null @@ -1,29 +0,0 @@ -# inbucket - disposable email service -# -# Inbucket is an SMTP server with a web interface for testing application -# functionality - -description "inbucket - disposable email service" -author "http://jhillyerd.github.com/inbucket" - -start on (local-filesystems and net-device-up IFACE!=lo) -stop on runlevel [!2345] - -env program=/opt/inbucket/inbucket -env config=/etc/opt/inbucket.conf -env logfile=/var/log/inbucket.log -env runas=inbucket - -# Give SMTP connections time to drain -kill timeout 20 - -pre-start script - [ -x $program ] - [ -r $config ] - touch $logfile - chown $runas: $logfile - # Allow bind to ports under 1024 - setcap 'cap_net_bind_service=+ep' $program -end script - -exec start-stop-daemon --start --chuid $runas --exec $program -- -logfile $logfile $config diff --git a/etc/ubuntu/inbucket.logrotate b/etc/ubuntu/inbucket.logrotate deleted file mode 100644 index 04ec439..0000000 --- a/etc/ubuntu/inbucket.logrotate +++ /dev/null @@ -1,8 +0,0 @@ -/var/log/inbucket.log { - missingok - notifempty - create 0644 inbucket inbucket - postrotate - [ -x /bin/systemctl ] && /bin/systemctl reload inbucket >/dev/null 2>&1 || true - endscript -} diff --git a/etc/ubuntu/inbucket.service b/etc/ubuntu/inbucket.service deleted file mode 100644 index 5480a45..0000000 --- a/etc/ubuntu/inbucket.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Inbucket Disposable Email Service -After=network.target - -[Service] -Type=simple -User=inbucket -Group=inbucket - -ExecStart=/opt/inbucket/inbucket -logfile /var/log/inbucket.log /etc/opt/inbucket.conf - -# Re-open log file after rotation -ExecReload=/bin/kill -HUP $MAINPID - -# Give SMTP connections time to drain -TimeoutStopSec=20 -KillMode=mixed - -[Install] -WantedBy=multi-user.target From 393a5b8d4e8a2ca14688691774ca1eecabeb5fc5 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 20:12:26 -0700 Subject: [PATCH 49/82] redhat: Use goreleaser to generate .rpm package for #89 --- .goreleaser.yml | 1 + CHANGELOG.md | 1 + etc/redhat/README | 3 - etc/redhat/httpd-vhost.conf | 17 ----- etc/redhat/inbucket-init.sh | 117 ---------------------------------- etc/redhat/inbucket.logrotate | 8 --- etc/redhat/inbucket.service | 20 ------ 7 files changed, 2 insertions(+), 165 deletions(-) delete mode 100644 etc/redhat/README delete mode 100644 etc/redhat/httpd-vhost.conf delete mode 100755 etc/redhat/inbucket-init.sh delete mode 100644 etc/redhat/inbucket.logrotate delete mode 100644 etc/redhat/inbucket.service diff --git a/.goreleaser.yml b/.goreleaser.yml index 864d60f..03764a7 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -61,6 +61,7 @@ nfpm: license: MIT formats: - deb + - rpm files: "ui/**/*": "/usr/local/share/inbucket/ui" config_files: diff --git a/CHANGELOG.md b/CHANGELOG.md index fd77963..888adb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Store size is now calculated during retention scan and displayed on the Status page. - Debian `.deb` package generation to release process. +- RedHat `.rpm` package generation to release process. ### Changed - Massive refactor of back-end code. Inbucket should now be both easier and diff --git a/etc/redhat/README b/etc/redhat/README deleted file mode 100644 index 26bac7a..0000000 --- a/etc/redhat/README +++ /dev/null @@ -1,3 +0,0 @@ -Please see the RedHat installation guide on our website: - -http://www.inbucket.org/installation/redhat.html diff --git a/etc/redhat/httpd-vhost.conf b/etc/redhat/httpd-vhost.conf deleted file mode 100644 index 02eff5c..0000000 --- a/etc/redhat/httpd-vhost.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Inbucket reverse proxy, Apache will forward requests from port 80 -# to Inbucket's built in web server on port 9000 -# -# Replace SERVERFQDN with your servers fully qualified domain name - - ServerName SERVERFQDN - ProxyRequests off - - - Order allow,deny - Allow from all - - - RewriteRule ^/$ http://SERVERFQDN:9000 - ProxyPass / http://SERVERFQDN:9000/ - ProxyPassReverse / http://SERVERFQDN:9000/ - diff --git a/etc/redhat/inbucket-init.sh b/etc/redhat/inbucket-init.sh deleted file mode 100755 index 504a3a7..0000000 --- a/etc/redhat/inbucket-init.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/sh -# -# inbucket Inbucket email testing service -# -# chkconfig: 2345 80 30 -# description: Inbucket is a disposable email service for testing email -# functionality of other applications. -# processname: inbucket -# pidfile: /var/run/inbucket/inbucket.pid - -### BEGIN INIT INFO -# Provides: Inbucket service -# Required-Start: $local_fs $network $remote_fs -# Required-Stop: $local_fs $network $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop inbucket -# Description: Inbucket is a disposable email service for testing email -# functionality of other applications. -# moves mail from one machine to another. -### END INIT INFO - -# Source function library. -. /etc/rc.d/init.d/functions - -# Source networking configuration. -. /etc/sysconfig/network - -RETVAL=0 -program=/opt/inbucket/inbucket -prog=${program##*/} -config=/etc/opt/inbucket.conf -runas=inbucket - -lockfile=/var/lock/subsys/$prog -pidfile=/var/run/$prog/$prog.pid -logfile=/var/log/$prog.log - -conf_check() { - [ -x $program ] || exit 5 - [ -f $config ] || exit 6 -} - -perms_check() { - mkdir -p /var/run/$prog - chown $runas: /var/run/$prog - touch $logfile - chown $runas: $logfile - # Allow bind to ports under 1024 - setcap 'cap_net_bind_service=+ep' $program -} - -start() { - [ "$EUID" != "0" ] && exit 4 - # Check that networking is up. - [ ${NETWORKING} = "no" ] && exit 1 - # Check config sanity - conf_check - perms_check - # Start daemon - echo -n $"Starting $prog: " - daemon --user $runas --pidfile $pidfile $program \ - -pidfile $pidfile -logfile $logfile $config \& - RETVAL=$? - [ $RETVAL -eq 0 ] && touch $lockfile - echo - return $RETVAL -} - -stop() { - [ "$EUID" != "0" ] && exit 4 - conf_check - # Stop daemon - echo -n $"Shutting down $prog: " - killproc -p "$pidfile" -d 15 "$program" - RETVAL=$? - [ $RETVAL -eq 0 ] && rm -f $lockfile $pidfile - echo - return $RETVAL -} - -reload() { - [ "$EUID" != "0" ] && exit 4 - echo -n $"Reloading $prog: " - killproc -p "$pidfile" "$program" -HUP - RETVAL=$? - echo - return $RETVAL -} - -# See how we were called. -case "$1" in - start) - [ -e $lockfile ] && exit 0 - start - ;; - stop) - [ -e $lockfile ] || exit 0 - stop - ;; - reload) - [ -e $lockfile ] || exit 0 - reload - ;; - restart|force-reload) - stop - start - ;; - status) - status -p $pidfile -l $(basename $lockfile) $prog - ;; - *) - echo $"Usage: $0 {start|stop|restart|status}" - exit 2 -esac - -exit $? diff --git a/etc/redhat/inbucket.logrotate b/etc/redhat/inbucket.logrotate deleted file mode 100644 index 04ec439..0000000 --- a/etc/redhat/inbucket.logrotate +++ /dev/null @@ -1,8 +0,0 @@ -/var/log/inbucket.log { - missingok - notifempty - create 0644 inbucket inbucket - postrotate - [ -x /bin/systemctl ] && /bin/systemctl reload inbucket >/dev/null 2>&1 || true - endscript -} diff --git a/etc/redhat/inbucket.service b/etc/redhat/inbucket.service deleted file mode 100644 index 5480a45..0000000 --- a/etc/redhat/inbucket.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Inbucket Disposable Email Service -After=network.target - -[Service] -Type=simple -User=inbucket -Group=inbucket - -ExecStart=/opt/inbucket/inbucket -logfile /var/log/inbucket.log /etc/opt/inbucket.conf - -# Re-open log file after rotation -ExecReload=/bin/kill -HUP $MAINPID - -# Give SMTP connections time to drain -TimeoutStopSec=20 -KillMode=mixed - -[Install] -WantedBy=multi-user.target From e2ba10c8cad2ca3d4f1b1d36e50ba720f71ad55f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 25 Mar 2018 21:57:23 -0700 Subject: [PATCH 50/82] Replace pkg/log with zerolog for normal logging #90 --- cmd/inbucket/main.go | 46 ++++++++++++---------- pkg/rest/apiv1_controller.go | 9 ----- pkg/rest/socketv1_controller.go | 33 ++++++++-------- pkg/server/pop3/listener.go | 41 ++++++++++---------- pkg/server/smtp/listener.go | 67 ++++++++++++++++----------------- pkg/server/web/helpers.go | 5 ++- pkg/server/web/server.go | 34 +++++++++++------ pkg/server/web/template.go | 14 ++++--- pkg/storage/file/fmessage.go | 11 ++++-- pkg/storage/file/fstore.go | 5 ++- pkg/storage/file/mbox.go | 22 ++++++----- pkg/storage/retention.go | 36 ++++++++++-------- pkg/webui/mailbox_controller.go | 7 ++-- 13 files changed, 177 insertions(+), 153 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 17ccb0e..9423128 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -13,7 +13,6 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/policy" @@ -25,6 +24,8 @@ import ( "github.com/jhillyerd/inbucket/pkg/storage/file" "github.com/jhillyerd/inbucket/pkg/storage/mem" "github.com/jhillyerd/inbucket/pkg/webui" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) var ( @@ -57,6 +58,7 @@ func main() { help := flag.Bool("help", false, "Displays help on flags and env variables.") pidfile := flag.String("pidfile", "", "Write our PID into the specified file.") logfile := flag.String("logfile", "stderr", "Write out log into the specified file.") + logjson := flag.Bool("logjson", false, "Logs are written in JSON format.") flag.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: inbucket [options]") flag.PrintDefaults() @@ -68,6 +70,17 @@ func main() { config.Usage() return } + // Logger setup. + if !*logjson { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } + _ = logfile + // } else if *logfile != "stderr" { + // // TODO #90 file output + // // defer close + // } + slog := log.With().Str("phase", "startup").Logger() // Process configuration. config.Version = version config.BuildDate = date @@ -80,23 +93,16 @@ func main() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) // Initialize logging. - log.SetLogLevel(conf.LogLevel) - if err := log.Initialize(*logfile); err != nil { - fmt.Fprintf(os.Stderr, "%v", err) - os.Exit(1) - } - defer log.Close() - log.Infof("Inbucket %v (%v) starting...", config.Version, config.BuildDate) + slog.Info().Str("version", config.Version).Str("buildDate", config.BuildDate).Msg("Inbucket") // Write pidfile if requested. if *pidfile != "" { pidf, err := os.Create(*pidfile) if err != nil { - log.Errorf("Failed to create %q: %v", *pidfile, err) - os.Exit(1) + slog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to create pidfile") } fmt.Fprintf(pidf, "%v\n", os.Getpid()) if err := pidf.Close(); err != nil { - log.Errorf("Failed to close PID file %q: %v", *pidfile, err) + slog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to close pidfile") } } // Configure internal services. @@ -104,9 +110,8 @@ func main() { shutdownChan := make(chan bool) store, err := storage.FromConfig(conf.Storage) if err != nil { - log.Errorf("Fatal storage error: %v", err) removePIDFile(*pidfile) - os.Exit(1) + slog.Fatal().Err(err).Str("module", "storage").Msg("Fatal storage error") } msgHub := msghub.New(rootCtx, conf.Web.MonitorHistory) addrPolicy := &policy.Addressing{Config: conf.SMTP} @@ -132,15 +137,17 @@ signalLoop: case sig := <-sigChan: switch sig { case syscall.SIGHUP: - log.Infof("Recieved SIGHUP, cycling logfile") - log.Rotate() + log.Info().Str("signal", "SIGHUP").Msg("Recieved SIGHUP, cycling logfile") + // TODO #90 log.Rotate() case syscall.SIGINT: // Shutdown requested - log.Infof("Received SIGINT, shutting down") + log.Info().Str("phase", "shutdown").Str("signal", "SIGINT"). + Msg("Received SIGINT, shutting down") close(shutdownChan) case syscall.SIGTERM: // Shutdown requested - log.Infof("Received SIGTERM, shutting down") + log.Info().Str("phase", "shutdown").Str("signal", "SIGTERM"). + Msg("Received SIGTERM, shutting down") close(shutdownChan) } case <-shutdownChan: @@ -160,7 +167,8 @@ signalLoop: func removePIDFile(pidfile string) { if pidfile != "" { if err := os.Remove(pidfile); err != nil { - log.Errorf("Failed to remove %q: %v", pidfile, err) + log.Error().Str("phase", "shutdown").Err(err).Str("path", pidfile). + Msg("Failed to remove pidfile") } } } @@ -168,7 +176,7 @@ func removePIDFile(pidfile string) { // timedExit is called as a goroutine during shutdown, it will force an exit after 15 seconds. func timedExit(pidfile string) { time.Sleep(15 * time.Second) - log.Errorf("Clean shutdown took too long, forcing exit") removePIDFile(pidfile) + log.Error().Str("phase", "shutdown").Msg("Clean shutdown took too long, forcing exit") os.Exit(0) } diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 34dea29..adef527 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "strconv" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -28,8 +27,6 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) } - log.Tracef("Got %v messsages", len(messages)) - jmessages := make([]*model.JSONMessageHeaderV1, len(messages)) for i, msg := range messages { jmessages[i] = &model.JSONMessageHeaderV1{ @@ -62,7 +59,6 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - attachParts := msg.Attachments() attachments := make([]*model.JSONMessageAttachmentV1, len(attachParts)) for i, part := range attachParts { @@ -78,7 +74,6 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( MD5: hex.EncodeToString(checksum[:]), } } - return web.RenderJSON(w, &model.JSONMessageV1{ Mailbox: name, @@ -109,8 +104,6 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } - log.Tracef("HTTP purged mailbox for %q", name) - return web.RenderJSON(w, "OK") } @@ -122,7 +115,6 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - r, err := ctx.Manager.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) @@ -155,6 +147,5 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) // This doesn't indicate missing, likely an IO error return fmt.Errorf("RemoveMessage(%q) failed: %v", id, err) } - return web.RenderJSON(w, "OK") } diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index d0ceddd..7c5b019 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -5,10 +5,10 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/rs/zerolog/log" ) const ( @@ -62,11 +62,13 @@ func (ml *msgListener) Receive(msg msghub.Message) error { // WSReader makes sure the websocket client is still connected, discards any messages from client func (ml *msgListener) WSReader(conn *websocket.Conn) { + slog := log.With().Str("module", "rest").Str("proto", "WebSocket"). + Str("remote", conn.RemoteAddr().String()).Logger() defer ml.Close() conn.SetReadLimit(maxMessageSize) conn.SetReadDeadline(time.Now().Add(pongWait)) conn.SetPongHandler(func(string) error { - log.Tracef("HTTP[%v] Got WebSocket pong", conn.RemoteAddr()) + slog.Debug().Msg("Got pong") conn.SetReadDeadline(time.Now().Add(pongWait)) return nil }) @@ -80,9 +82,9 @@ func (ml *msgListener) WSReader(conn *websocket.Conn) { websocket.CloseNoStatusReceived, ) { // Unexpected close code - log.Warnf("HTTP[%v] WebSocket error: %v", conn.RemoteAddr(), err) + slog.Warn().Err(err).Msg("Socket error") } else { - log.Tracef("HTTP[%v] Closing WebSocket", conn.RemoteAddr()) + slog.Debug().Msg("Closing socket") } break } @@ -127,7 +129,8 @@ func (ml *msgListener) WSWriter(conn *websocket.Conn) { // Write error return } - log.Tracef("HTTP[%v] Sent WebSocket ping", conn.RemoteAddr()) + log.Debug().Str("module", "rest").Str("proto", "WebSocket"). + Str("remote", conn.RemoteAddr().String()).Msg("Sent ping") } } } @@ -147,7 +150,7 @@ func (ml *msgListener) Close() { // the client of all messages received. func MonitorAllMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - // Upgrade to Websocket + // Upgrade to Websocket. conn, err := upgrader.Upgrade(w, req, nil) if err != nil { return err @@ -157,14 +160,12 @@ func MonitorAllMessagesV1( _ = conn.Close() web.ExpWebSocketConnectsCurrent.Add(-1) }() - - log.Tracef("HTTP[%v] Upgraded to websocket", req.RemoteAddr) - - // Create, register listener; then interact with conn + log.Debug().Str("module", "rest").Str("proto", "WebSocket"). + Str("remote", conn.RemoteAddr().String()).Msg("Upgraded to WebSocket") + // Create, register listener; then interact with conn. ml := newMsgListener(ctx.MsgHub, "") go ml.WSWriter(conn) ml.WSReader(conn) - return nil } @@ -176,7 +177,7 @@ func MonitorMailboxMessagesV1( if err != nil { return err } - // Upgrade to Websocket + // Upgrade to Websocket. conn, err := upgrader.Upgrade(w, req, nil) if err != nil { return err @@ -186,13 +187,11 @@ func MonitorMailboxMessagesV1( _ = conn.Close() web.ExpWebSocketConnectsCurrent.Add(-1) }() - - log.Tracef("HTTP[%v] Upgraded to websocket", req.RemoteAddr) - - // Create, register listener; then interact with conn + log.Debug().Str("module", "rest").Str("proto", "WebSocket"). + Str("remote", conn.RemoteAddr().String()).Msg("Upgraded to WebSocket") + // Create, register listener; then interact with conn. ml := newMsgListener(ctx.MsgHub, name) go ml.WSWriter(conn) ml.WSReader(conn) - return nil } diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index 2db2d00..bf9d6fd 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -7,8 +7,8 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/rs/zerolog/log" ) // Server defines an instance of our POP3 server @@ -36,44 +36,42 @@ func New(cfg config.POP3, shutdownChan chan bool, store storage.Store) *Server { // Start the server and listen for connections func (s *Server) Start(ctx context.Context) { + slog := log.With().Str("module", "pop3").Str("phase", "startup").Logger() addr, err := net.ResolveTCPAddr("tcp4", s.host) if err != nil { - log.Errorf("POP3 Failed to build tcp4 address: %v", err) + slog.Error().Err(err).Msg("Failed to build tcp4 address") s.emergencyShutdown() return } - - log.Infof("POP3 listening on TCP4 %v", addr) + slog.Info().Str("addr", addr.String()).Msg("POP3 listening on tcp4") s.listener, err = net.ListenTCP("tcp4", addr) if err != nil { - log.Errorf("POP3 failed to start tcp4 listener: %v", err) + slog.Error().Err(err).Msg("Failed to start tcp4 listener") s.emergencyShutdown() return } - - // Listener go routine + // Listener go routine. go s.serve(ctx) - - // Wait for shutdown + // Wait for shutdown. select { case _ = <-ctx.Done(): } - - log.Tracef("POP3 shutdown requested, connections will be drained") - // Closing the listener will cause the serve() go routine to exit + slog = log.With().Str("module", "pop3").Str("phase", "shutdown").Logger() + slog.Debug().Msg("POP3 shutdown requested, connections will be drained") + // Closing the listener will cause the serve() go routine to exit. if err := s.listener.Close(); err != nil { - log.Errorf("Error closing POP3 listener: %v", err) + slog.Error().Err(err).Msg("Failed to close POP3 listener") } } -// serve is the listen/accept loop +// serve is the listen/accept loop. func (s *Server) serve(ctx context.Context) { - // Handle incoming connections + // Handle incoming connections. var tempDelay time.Duration for sid := 1; ; sid++ { if conn, err := s.listener.Accept(); err != nil { if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - // Temporary error, sleep for a bit and try again + // Temporary error, sleep for a bit and try again. if tempDelay == 0 { tempDelay = 5 * time.Millisecond } else { @@ -82,17 +80,18 @@ func (s *Server) serve(ctx context.Context) { if max := 1 * time.Second; tempDelay > max { tempDelay = max } - log.Errorf("POP3 accept error: %v; retrying in %v", err, tempDelay) + log.Error().Str("module", "pop3").Err(err). + Msgf("POP3 accept error; retrying in %v", tempDelay) time.Sleep(tempDelay) continue } else { - // Permanent error + // Permanent error. select { case <-ctx.Done(): - // POP3 is shutting down + // POP3 is shutting down. return default: - // Something went wrong + // Something went wrong. s.emergencyShutdown() return } @@ -118,5 +117,5 @@ func (s *Server) emergencyShutdown() { func (s *Server) Drain() { // Wait for sessions to close s.waitgroup.Wait() - log.Tracef("POP3 connections have drained") + log.Debug().Str("module", "pop3").Str("phase", "shutdown").Msg("POP3 connections have drained") } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 339e32e..d566e15 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -10,9 +10,9 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/policy" + "github.com/rs/zerolog/log" ) func init() { @@ -27,12 +27,13 @@ func init() { m.Set("WarnsTotal", expWarnsTotal) m.Set("WarnsHist", expWarnsHist) - log.AddTickerFunc(func() { - expReceivedHist.Set(log.PushMetric(deliveredHist, expReceivedTotal)) - expConnectsHist.Set(log.PushMetric(connectsHist, expConnectsTotal)) - expErrorsHist.Set(log.PushMetric(errorsHist, expErrorsTotal)) - expWarnsHist.Set(log.PushMetric(warnsHist, expWarnsTotal)) - }) + // TODO #90 move elsewhere + // log.AddTickerFunc(func() { + // expReceivedHist.Set(log.PushMetric(deliveredHist, expReceivedTotal)) + // expConnectsHist.Set(log.PushMetric(connectsHist, expConnectsTotal)) + // expErrorsHist.Set(log.PushMetric(errorsHist, expErrorsTotal)) + // expWarnsHist.Set(log.PushMetric(warnsHist, expWarnsTotal)) + // }) } // Server holds the configuration and state of our SMTP server @@ -99,51 +100,48 @@ func NewServer( } } -// Start the listener and handle incoming connections +// Start the listener and handle incoming connections. func (s *Server) Start(ctx context.Context) { + slog := log.With().Str("module", "smtp").Str("phase", "startup").Logger() addr, err := net.ResolveTCPAddr("tcp4", s.host) if err != nil { - log.Errorf("Failed to build tcp4 address: %v", err) + slog.Error().Err(err).Msg("Failed to build tcp4 address") s.emergencyShutdown() return } - - log.Infof("SMTP listening on TCP4 %v", addr) + slog.Info().Str("addr", addr.String()).Msg("SMTP listening on tcp4") s.listener, err = net.ListenTCP("tcp4", addr) if err != nil { - log.Errorf("SMTP failed to start tcp4 listener: %v", err) + slog.Error().Err(err).Msg("Failed to start tcp4 listener") s.emergencyShutdown() return } - if !s.storeMessages { - log.Infof("Load test mode active, messages will not be stored") + slog.Info().Msg("Load test mode active, messages will not be stored") } else if s.domainNoStore != "" { - log.Infof("Messages sent to domain '%v' will be discarded", s.domainNoStore) + slog.Info().Msgf("Messages sent to domain '%v' will be discarded", s.domainNoStore) } - - // Listener go routine + // Listener go routine. go s.serve(ctx) - - // Wait for shutdown + // Wait for shutdown. <-ctx.Done() - log.Tracef("SMTP shutdown requested, connections will be drained") - - // Closing the listener will cause the serve() go routine to exit + slog = log.With().Str("module", "smtp").Str("phase", "shutdown").Logger() + slog.Debug().Msg("SMTP shutdown requested, connections will be drained") + // Closing the listener will cause the serve() go routine to exit. if err := s.listener.Close(); err != nil { - log.Errorf("Failed to close SMTP listener: %v", err) + slog.Error().Err(err).Msg("Failed to close SMTP listener") } } -// serve is the listen/accept loop +// serve is the listen/accept loop. func (s *Server) serve(ctx context.Context) { - // Handle incoming connections + // Handle incoming connections. var tempDelay time.Duration for sessionID := 1; ; sessionID++ { if conn, err := s.listener.Accept(); err != nil { - // There was an error accepting the connection + // There was an error accepting the connection. if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - // Temporary error, sleep for a bit and try again + // Temporary error, sleep for a bit and try again. if tempDelay == 0 { tempDelay = 5 * time.Millisecond } else { @@ -152,17 +150,18 @@ func (s *Server) serve(ctx context.Context) { if max := 1 * time.Second; tempDelay > max { tempDelay = max } - log.Errorf("SMTP accept error: %v; retrying in %v", err, tempDelay) + log.Error().Str("module", "smtp").Err(err). + Msgf("SMTP accept error; retrying in %v", tempDelay) time.Sleep(tempDelay) continue } else { - // Permanent error + // Permanent error. select { case <-ctx.Done(): - // SMTP is shutting down + // SMTP is shutting down. return default: - // Something went wrong + // Something went wrong. s.emergencyShutdown() return } @@ -177,7 +176,7 @@ func (s *Server) serve(ctx context.Context) { } func (s *Server) emergencyShutdown() { - // Shutdown Inbucket + // Shutdown Inbucket. select { case <-s.globalShutdown: default: @@ -187,7 +186,7 @@ func (s *Server) emergencyShutdown() { // Drain causes the caller to block until all active SMTP sessions have finished func (s *Server) Drain() { - // Wait for sessions to close + // Wait for sessions to close. s.waitgroup.Wait() - log.Tracef("SMTP connections have drained") + log.Debug().Str("module", "smtp").Str("phase", "shutdown").Msg("SMTP connections have drained") } diff --git a/pkg/server/web/helpers.go b/pkg/server/web/helpers.go index c41e1e6..60d2edc 100644 --- a/pkg/server/web/helpers.go +++ b/pkg/server/web/helpers.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/pkg/log" + "github.com/rs/zerolog/log" ) // TemplateFuncs declares functions made available to all templates (including partials) @@ -42,7 +42,8 @@ func Reverse(name string, things ...interface{}) string { // Grab the route u, err := Router.Get(name).URL(strs...) if err != nil { - log.Errorf("Failed to reverse route: %v", err) + log.Error().Str("module", "web").Str("name", name).Err(err). + Msg("Failed to reverse route") return "/ROUTE-ERROR" } return u.Path diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index 8e5524c..a60735a 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -13,9 +13,9 @@ import ( "github.com/gorilla/securecookie" "github.com/gorilla/sessions" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/rs/zerolog/log" ) // Handler is a function type that handles an HTTP request in Inbucket @@ -66,17 +66,20 @@ func Initialize( // Content Paths staticPath := filepath.Join(conf.Web.UIDir, staticDir) - log.Infof("Web UI content mapped to path: %s", conf.Web.UIDir) + log.Info().Str("module", "web").Str("phase", "startup").Str("path", conf.Web.UIDir). + Msg("Web UI content mapped") Router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", http.FileServer(http.Dir(staticPath)))) http.Handle("/", Router) // Session cookie setup if conf.Web.CookieAuthKey == "" { - log.Infof("HTTP generating random cookie.auth.key") + log.Info().Str("module", "web").Str("phase", "startup"). + Msg("Generating random cookie.auth.key") sessionStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64)) } else { - log.Tracef("HTTP using configured cookie.auth.key") + log.Info().Str("module", "web").Str("phase", "startup"). + Msg("Using configured cookie.auth.key") sessionStore = sessions.NewCookieStore([]byte(conf.Web.CookieAuthKey)) } } @@ -91,11 +94,13 @@ func Start(ctx context.Context) { } // We don't use ListenAndServe because it lacks a way to close the listener - log.Infof("HTTP listening on TCP4 %v", server.Addr) + log.Info().Str("module", "web").Str("phase", "startup").Str("addr", server.Addr). + Msg("HTTP listening on tcp4") var err error listener, err = net.Listen("tcp", server.Addr) if err != nil { - log.Errorf("HTTP failed to start TCP4 listener: %v", err) + log.Error().Str("module", "web").Str("phase", "startup").Err(err). + Msg("HTTP failed to start TCP4 listener") emergencyShutdown() return } @@ -106,12 +111,14 @@ func Start(ctx context.Context) { // Wait for shutdown select { case _ = <-ctx.Done(): - log.Tracef("HTTP server shutting down on request") + log.Debug().Str("module", "web").Str("phase", "shutdown"). + Msg("HTTP server shutting down on request") } // Closing the listener will cause the serve() go routine to exit if err := listener.Close(); err != nil { - log.Errorf("Failed to close HTTP listener: %v", err) + log.Debug().Str("module", "web").Str("phase", "shutdown").Err(err). + Msg("Failed to close HTTP listener") } } @@ -124,7 +131,8 @@ func serve(ctx context.Context) { case _ = <-ctx.Done(): // Nop default: - log.Errorf("HTTP server failed: %v", err) + log.Error().Str("module", "web").Str("phase", "startup").Err(err). + Msg("HTTP server failed") emergencyShutdown() return } @@ -135,17 +143,19 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Create the context ctx, err := NewContext(req) if err != nil { - log.Errorf("HTTP failed to create context: %v", err) + log.Error().Str("module", "web").Err(err).Msg("HTTP failed to create context") http.Error(w, err.Error(), http.StatusInternalServerError) return } defer ctx.Close() // Run the handler, grab the error, and report it - log.Tracef("HTTP[%v] %v %v %q", req.RemoteAddr, req.Proto, req.Method, req.RequestURI) + log.Debug().Str("module", "web").Str("remote", req.RemoteAddr).Str("proto", req.Proto). + Str("method", req.Method).Str("path", req.RequestURI).Msg("Request") err = h(w, req, ctx) if err != nil { - log.Errorf("HTTP error handling %q: %v", req.RequestURI, err) + log.Error().Str("module", "web").Str("path", req.RequestURI).Err(err). + Msg("Error handling request") http.Error(w, err.Error(), http.StatusInternalServerError) return } diff --git a/pkg/server/web/template.go b/pkg/server/web/template.go index 87675a5..f9f8da2 100644 --- a/pkg/server/web/template.go +++ b/pkg/server/web/template.go @@ -7,7 +7,7 @@ import ( "path/filepath" "sync" - "github.com/jhillyerd/inbucket/pkg/log" + "github.com/rs/zerolog/log" ) var cachedMutex sync.Mutex @@ -19,7 +19,8 @@ var cachedPartials = map[string]*template.Template{} func RenderTemplate(name string, w http.ResponseWriter, data interface{}) error { t, err := ParseTemplate(name, false) if err != nil { - log.Errorf("Error in template '%v': %v", name, err) + log.Error().Str("module", "web").Str("path", name).Err(err). + Msg("Error in template") return err } w.Header().Set("Expires", "-1") @@ -31,7 +32,8 @@ func RenderTemplate(name string, w http.ResponseWriter, data interface{}) error func RenderPartial(name string, w http.ResponseWriter, data interface{}) error { t, err := ParseTemplate(name, true) if err != nil { - log.Errorf("Error in template '%v': %v", name, err) + log.Error().Str("module", "web").Str("path", name).Err(err). + Msg("Error in template") return err } w.Header().Set("Expires", "-1") @@ -49,7 +51,7 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { } tempFile := filepath.Join(rootConfig.Web.UIDir, templateDir, filepath.FromSlash(name)) - log.Tracef("Parsing template %v", tempFile) + log.Debug().Str("module", "web").Str("path", name).Msg("Parsing template") var err error var t *template.Template @@ -70,10 +72,10 @@ func ParseTemplate(name string, partial bool) (*template.Template, error) { // Allows us to disable caching for theme development if rootConfig.Web.TemplateCache { if partial { - log.Tracef("Caching partial %v", name) + log.Debug().Str("module", "web").Str("path", name).Msg("Caching partial") cachedTemplates[name] = t } else { - log.Tracef("Caching template %v", name) + log.Debug().Str("module", "web").Str("path", name).Msg("Caching template") cachedTemplates[name] = t } } diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 79d1a65..961df62 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -7,7 +7,7 @@ import ( "path/filepath" "time" - "github.com/jhillyerd/inbucket/pkg/log" + "github.com/rs/zerolog/log" ) // Message implements Message and contains a little bit of data about a @@ -35,9 +35,12 @@ func (mb *mbox) newMessage() (*Message, error) { // Delete old messages over messageCap if mb.store.messageCap > 0 { for len(mb.messages) >= mb.store.messageCap { - log.Infof("Mailbox %q over configured message cap", mb.name) - if err := mb.removeMessage(mb.messages[0].ID()); err != nil { - log.Errorf("Error deleting message: %s", err) + log.Info().Str("module", "storage").Str("mailbox", mb.name). + Msg("Mailbox over message cap") + id := mb.messages[0].ID() + if err := mb.removeMessage(id); err != nil { + log.Error().Str("module", "storage").Str("mailbox", mb.name).Str("id", id). + Err(err).Msg("Unable to delete message") } } } diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index fa00f3c..75ec3ba 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -10,10 +10,10 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" + "github.com/rs/zerolog/log" ) // Name of index file in each mailbox @@ -57,7 +57,8 @@ func New(cfg config.Storage) (storage.Store, error) { if _, err := os.Stat(mailPath); err != nil { // Mail datastore does not yet exist if err = os.MkdirAll(mailPath, 0770); err != nil { - log.Errorf("Error creating dir %q: %v", mailPath, err) + log.Error().Str("module", "storage").Str("path", mailPath).Err(err). + Msg("Error creating dir") } } return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap}, nil diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index 9a59984..8c85145 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -9,8 +9,8 @@ import ( "path/filepath" "sync" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/rs/zerolog/log" ) // mbox manages the mail for a specific user and correlates to a particular directory on disk. @@ -87,7 +87,7 @@ func (mb *mbox) removeMessage(id string) error { return nil } // There are still messages in the index - log.Tracef("Deleting %v", msg.rawPath()) + log.Debug().Str("module", "storage").Str("path", msg.rawPath()).Msg("Deleting file") return os.Remove(msg.rawPath()) } @@ -104,7 +104,8 @@ func (mb *mbox) readIndex() error { // Check if index exists if _, err := os.Stat(mb.indexPath); err != nil { // Does not exist, but that's not an error in our world - log.Tracef("Index %v does not exist (yet)", mb.indexPath) + log.Debug().Str("module", "storage").Str("path", mb.indexPath). + Msg("Index does not yet exist") mb.indexLoaded = true return nil } @@ -114,7 +115,8 @@ func (mb *mbox) readIndex() error { } defer func() { if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) + log.Error().Str("module", "storage").Str("path", mb.indexPath).Err(err). + Msg("Failed to close") } }() // Decode gob data @@ -171,12 +173,13 @@ func (mb *mbox) writeIndex() error { return err } if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) + log.Error().Str("module", "storage").Str("path", mb.indexPath).Err(err). + Msg("Failed to close") return err } } else { // No messages, delete index+maildir - log.Tracef("Removing mailbox %v", mb.path) + log.Debug().Str("module", "storage").Str("path", mb.path).Msg("Removing mailbox") return mb.removeDir() } return nil @@ -186,7 +189,8 @@ func (mb *mbox) writeIndex() error { func (mb *mbox) createDir() error { if _, err := os.Stat(mb.path); err != nil { if err := os.MkdirAll(mb.path, 0770); err != nil { - log.Errorf("Failed to create directory %v, %v", mb.path, err) + log.Error().Str("module", "storage").Str("path", mb.path).Err(err). + Msg("Failed to create directory") return err } } @@ -223,10 +227,10 @@ func removeDirIfEmpty(path string) (removed bool) { // Dir not empty return false } - log.Tracef("Removing dir %v", path) + log.Debug().Str("module", "storage").Str("path", path).Msg("Removing dir") err = os.Remove(path) if err != nil { - log.Errorf("Failed to remove %q: %v", path, err) + log.Error().Str("module", "storage").Str("path", path).Err(err).Msg("Failed to remove") return false } return true diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index c17e725..6a0b714 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -7,7 +7,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/log" + "github.com/rs/zerolog/log" ) var ( @@ -42,11 +42,12 @@ func init() { rm.Set("RetainedSize", expRetainedSize) rm.Set("SizeHist", expSizeHist) - log.AddTickerFunc(func() { - expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) - expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) - expSizeHist.Set(log.PushMetric(sizeHist, expRetainedSize)) - }) + // TODO #90 move + // log.AddTickerFunc(func() { + // expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) + // expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) + // expSizeHist.Set(log.PushMetric(sizeHist, expRetainedSize)) + // }) } // RetentionScanner looks for messages older than the configured retention period and deletes them. @@ -79,16 +80,18 @@ func NewRetentionScanner( // Start up the retention scanner if retention period > 0 func (rs *RetentionScanner) Start() { if rs.retentionPeriod <= 0 { - log.Infof("Retention scanner disabled") + log.Info().Str("phase", "startup").Str("module", "storage").Msg("Retention scanner disabled") close(rs.retentionShutdown) return } - log.Infof("Retention configured for %v", rs.retentionPeriod) + log.Info().Str("phase", "startup").Str("module", "storage"). + Msgf("Retention configured for %v", rs.retentionPeriod) go rs.run() } // run loops to kick off the scanner on the correct schedule func (rs *RetentionScanner) run() { + slog := log.With().Str("module", "storage").Logger() start := time.Now() retentionLoop: for { @@ -96,7 +99,7 @@ retentionLoop: since := time.Since(start) if since < time.Minute { dur := time.Minute - since - log.Tracef("Retention scanner sleeping for %v", dur) + slog.Debug().Msgf("Retention scanner sleeping for %v", dur) select { case <-rs.globalShutdown: break retentionLoop @@ -106,7 +109,7 @@ retentionLoop: // Kickoff scan start = time.Now() if err := rs.DoScan(); err != nil { - log.Errorf("Error during retention scan: %v", err) + slog.Error().Err(err).Msg("Error during retention scan") } // Check for global shutdown select { @@ -115,13 +118,14 @@ retentionLoop: default: } } - log.Tracef("Retention scanner shut down") + slog.Debug().Str("phase", "shutdown").Msg("Retention scanner shut down") close(rs.retentionShutdown) } // DoScan does a single pass of all mailboxes looking for messages that can be purged. func (rs *RetentionScanner) DoScan() error { - log.Tracef("Starting retention scan") + slog := log.With().Str("module", "storage").Logger() + slog.Debug().Msg("Starting retention scan") cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 storeSize := int64(0) @@ -129,9 +133,11 @@ func (rs *RetentionScanner) DoScan() error { err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { - log.Tracef("Purging expired message %v/%v", msg.Mailbox(), msg.ID()) + slog.Debug().Str("mailbox", msg.Mailbox()). + Msgf("Purging expired message %v", msg.ID()) if err := rs.ds.RemoveMessage(msg.Mailbox(), msg.ID()); err != nil { - log.Errorf("Failed to purge message %v: %v", msg.ID(), err) + slog.Error().Str("mailbox", msg.Mailbox()).Err(err). + Msgf("Failed to purge message %v", msg.ID()) } else { expRetentionDeletesTotal.Add(1) } @@ -142,7 +148,7 @@ func (rs *RetentionScanner) DoScan() error { } select { case <-rs.globalShutdown: - log.Tracef("Retention scan aborted due to shutdown") + slog.Debug().Str("phase", "shutdown").Msg("Retention scan aborted due to shutdown") return false case <-time.After(rs.retentionSleep): // Reduce disk thrashing diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index e835d78..199b5e4 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -7,10 +7,10 @@ import ( "net/http" "strconv" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/webui/sanitize" + "github.com/rs/zerolog/log" ) // MailboxIndex renders the index page for a particular mailbox @@ -76,7 +76,6 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) } - log.Tracef("Got %v messsages", len(messages)) // Render partial template return web.RenderPartial("mailbox/_list.html", w, map[string]interface{}{ "ctx": ctx, @@ -109,7 +108,9 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if str, err := sanitize.HTML(msg.HTML()); err == nil { htmlBody = template.HTML(str) } else { - log.Warnf("HTML sanitizer failed: %s", err) + // Soft failure, render empty tab. + log.Warn().Str("module", "webui").Str("mailbox", name).Str("id", id).Err(err). + Msg("HTML sanitizer failed") } } // Render partial template From 6f25a1320e18c306741d9b652ec231ef71482681 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 27 Mar 2018 20:44:17 -0700 Subject: [PATCH 51/82] pop3, smtp: rename Session method receivers to s --- pkg/server/pop3/handler.go | 416 ++++++++++++++++++------------------- pkg/server/smtp/handler.go | 294 +++++++++++++------------- 2 files changed, 355 insertions(+), 355 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 72d4355..61384d7 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -78,8 +78,8 @@ func NewSession(server *Server, id int, conn net.Conn) *Session { reader: reader, remoteHost: host} } -func (ses *Session) String() string { - return fmt.Sprintf("Session{id: %v, state: %v}", ses.id, ses.state) +func (s *Session) String() string { + return fmt.Sprintf("Session{id: %v, state: %v}", s.id, s.state) } /* Session flow: @@ -100,23 +100,23 @@ func (s *Server) startSession(id int, conn net.Conn) { //expConnectsCurrent.Add(-1) }() - ses := NewSession(s, id, conn) - ses.send(fmt.Sprintf("+OK Inbucket POP3 server ready <%v.%v@%v>", os.Getpid(), + ssn := NewSession(s, id, conn) + ssn.send(fmt.Sprintf("+OK Inbucket POP3 server ready <%v.%v@%v>", os.Getpid(), time.Now().Unix(), s.domain)) // This is our command reading loop - for ses.state != QUIT && ses.sendError == nil { - line, err := ses.readLine() + for ssn.state != QUIT && ssn.sendError == nil { + line, err := ssn.readLine() if err == nil { - if cmd, arg, ok := ses.parseCmd(line); ok { + if cmd, arg, ok := ssn.parseCmd(line); ok { // Check against valid SMTP commands if cmd == "" { - ses.send("-ERR Speak up") + ssn.send("-ERR Speak up") continue } if !commands[cmd] { - ses.send(fmt.Sprintf("-ERR Syntax error, %v command unrecognized", cmd)) - ses.logWarn("Unrecognized command: %v", cmd) + ssn.send(fmt.Sprintf("-ERR Syntax error, %v command unrecognized", cmd)) + ssn.logWarn("Unrecognized command: %v", cmd) continue } @@ -124,307 +124,307 @@ func (s *Server) startSession(id int, conn net.Conn) { switch cmd { case "CAPA": // List our capabilities per RFC2449 - ses.send("+OK Capability list follows") - ses.send("TOP") - ses.send("USER") - ses.send("UIDL") - ses.send("IMPLEMENTATION Inbucket") - ses.send(".") + ssn.send("+OK Capability list follows") + ssn.send("TOP") + ssn.send("USER") + ssn.send("UIDL") + ssn.send("IMPLEMENTATION Inbucket") + ssn.send(".") continue } // Send command to handler for current state - switch ses.state { + switch ssn.state { case AUTHORIZATION: - ses.authorizationHandler(cmd, arg) + ssn.authorizationHandler(cmd, arg) continue case TRANSACTION: - ses.transactionHandler(cmd, arg) + ssn.transactionHandler(cmd, arg) continue } - ses.logError("Session entered unexpected state %v", ses.state) + ssn.logError("Session entered unexpected state %v", ssn.state) break } else { - ses.send("-ERR Syntax error, command garbled") + ssn.send("-ERR Syntax error, command garbled") } } else { // readLine() returned an error if err == io.EOF { - switch ses.state { + switch ssn.state { case AUTHORIZATION: // EOF is common here - ses.logInfo("Client closed connection (state %v)", ses.state) + ssn.logInfo("Client closed connection (state %v)", ssn.state) default: - ses.logWarn("Got EOF while in state %v", ses.state) + ssn.logWarn("Got EOF while in state %v", ssn.state) } break } // not an EOF - ses.logWarn("Connection error: %v", err) + ssn.logWarn("Connection error: %v", err) if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { - ses.send("-ERR Idle timeout, bye bye") + ssn.send("-ERR Idle timeout, bye bye") break } } - ses.send("-ERR Connection error, sorry") + ssn.send("-ERR Connection error, sorry") break } } - if ses.sendError != nil { - ses.logWarn("Network send error: %v", ses.sendError) + if ssn.sendError != nil { + ssn.logWarn("Network send error: %v", ssn.sendError) } - ses.logInfo("Closing connection") + ssn.logInfo("Closing connection") } // AUTHORIZATION state -func (ses *Session) authorizationHandler(cmd string, args []string) { +func (s *Session) authorizationHandler(cmd string, args []string) { switch cmd { case "QUIT": - ses.send("+OK Goodnight and good luck") - ses.enterState(QUIT) + s.send("+OK Goodnight and good luck") + s.enterState(QUIT) case "USER": if len(args) > 0 { - ses.user = args[0] - ses.send(fmt.Sprintf("+OK Hello %v, welcome to Inbucket", ses.user)) + s.user = args[0] + s.send(fmt.Sprintf("+OK Hello %v, welcome to Inbucket", s.user)) } else { - ses.send("-ERR Missing username argument") + s.send("-ERR Missing username argument") } case "PASS": - if ses.user == "" { - ses.ooSeq(cmd) + if s.user == "" { + s.ooSeq(cmd) } else { - ses.loadMailbox() - ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) - ses.enterState(TRANSACTION) + s.loadMailbox() + s.send(fmt.Sprintf("+OK Found %v messages for %v", s.msgCount, s.user)) + s.enterState(TRANSACTION) } case "APOP": if len(args) != 2 { - ses.logWarn("Expected two arguments for APOP") - ses.send("-ERR APOP requires two arguments") + s.logWarn("Expected two arguments for APOP") + s.send("-ERR APOP requires two arguments") return } - ses.user = args[0] - ses.loadMailbox() - ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) - ses.enterState(TRANSACTION) + s.user = args[0] + s.loadMailbox() + s.send(fmt.Sprintf("+OK Found %v messages for %v", s.msgCount, s.user)) + s.enterState(TRANSACTION) default: - ses.ooSeq(cmd) + s.ooSeq(cmd) } } // TRANSACTION state -func (ses *Session) transactionHandler(cmd string, args []string) { +func (s *Session) transactionHandler(cmd string, args []string) { switch cmd { case "STAT": if len(args) != 0 { - ses.logWarn("STAT got an unexpected argument") - ses.send("-ERR STAT command must have no arguments") + s.logWarn("STAT got an unexpected argument") + s.send("-ERR STAT command must have no arguments") return } var count int var size int64 - for i, msg := range ses.messages { - if ses.retain[i] { + for i, msg := range s.messages { + if s.retain[i] { count++ size += msg.Size() } } - ses.send(fmt.Sprintf("+OK %v %v", count, size)) + s.send(fmt.Sprintf("+OK %v %v", count, size)) case "LIST": if len(args) > 1 { - ses.logWarn("LIST command had more than 1 argument") - ses.send("-ERR LIST command must have zero or one argument") + s.logWarn("LIST command had more than 1 argument") + s.send("-ERR LIST command must have zero or one argument") return } if len(args) == 1 { msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - ses.logWarn("LIST command argument was not an integer") - ses.send("-ERR LIST command requires an integer argument") + s.logWarn("LIST command argument was not an integer") + s.send("-ERR LIST command requires an integer argument") return } if msgNum < 1 { - ses.logWarn("LIST command argument was less than 1") - ses.send("-ERR LIST argument must be greater than 0") + s.logWarn("LIST command argument was less than 1") + s.send("-ERR LIST argument must be greater than 0") return } - if int(msgNum) > len(ses.messages) { - ses.logWarn("LIST command argument was greater than number of messages") - ses.send("-ERR LIST argument must not exceed the number of messages") + if int(msgNum) > len(s.messages) { + s.logWarn("LIST command argument was greater than number of messages") + s.send("-ERR LIST argument must not exceed the number of messages") return } - if !ses.retain[msgNum-1] { - ses.logWarn("Client tried to LIST a message it had deleted") - ses.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) + if !s.retain[msgNum-1] { + s.logWarn("Client tried to LIST a message it had deleted") + s.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) return } - ses.send(fmt.Sprintf("+OK %v %v", msgNum, ses.messages[msgNum-1].Size())) + s.send(fmt.Sprintf("+OK %v %v", msgNum, s.messages[msgNum-1].Size())) } else { - ses.send(fmt.Sprintf("+OK Listing %v messages", ses.msgCount)) - for i, msg := range ses.messages { - if ses.retain[i] { - ses.send(fmt.Sprintf("%v %v", i+1, msg.Size())) + s.send(fmt.Sprintf("+OK Listing %v messages", s.msgCount)) + for i, msg := range s.messages { + if s.retain[i] { + s.send(fmt.Sprintf("%v %v", i+1, msg.Size())) } } - ses.send(".") + s.send(".") } case "UIDL": if len(args) > 1 { - ses.logWarn("UIDL command had more than 1 argument") - ses.send("-ERR UIDL command must have zero or one argument") + s.logWarn("UIDL command had more than 1 argument") + s.send("-ERR UIDL command must have zero or one argument") return } if len(args) == 1 { msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - ses.logWarn("UIDL command argument was not an integer") - ses.send("-ERR UIDL command requires an integer argument") + s.logWarn("UIDL command argument was not an integer") + s.send("-ERR UIDL command requires an integer argument") return } if msgNum < 1 { - ses.logWarn("UIDL command argument was less than 1") - ses.send("-ERR UIDL argument must be greater than 0") + s.logWarn("UIDL command argument was less than 1") + s.send("-ERR UIDL argument must be greater than 0") return } - if int(msgNum) > len(ses.messages) { - ses.logWarn("UIDL command argument was greater than number of messages") - ses.send("-ERR UIDL argument must not exceed the number of messages") + if int(msgNum) > len(s.messages) { + s.logWarn("UIDL command argument was greater than number of messages") + s.send("-ERR UIDL argument must not exceed the number of messages") return } - if !ses.retain[msgNum-1] { - ses.logWarn("Client tried to UIDL a message it had deleted") - ses.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) + if !s.retain[msgNum-1] { + s.logWarn("Client tried to UIDL a message it had deleted") + s.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) return } - ses.send(fmt.Sprintf("+OK %v %v", msgNum, ses.messages[msgNum-1].ID())) + s.send(fmt.Sprintf("+OK %v %v", msgNum, s.messages[msgNum-1].ID())) } else { - ses.send(fmt.Sprintf("+OK Listing %v messages", ses.msgCount)) - for i, msg := range ses.messages { - if ses.retain[i] { - ses.send(fmt.Sprintf("%v %v", i+1, msg.ID())) + s.send(fmt.Sprintf("+OK Listing %v messages", s.msgCount)) + for i, msg := range s.messages { + if s.retain[i] { + s.send(fmt.Sprintf("%v %v", i+1, msg.ID())) } } - ses.send(".") + s.send(".") } case "DELE": if len(args) != 1 { - ses.logWarn("DELE command had invalid number of arguments") - ses.send("-ERR DELE command requires a single argument") + s.logWarn("DELE command had invalid number of arguments") + s.send("-ERR DELE command requires a single argument") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - ses.logWarn("DELE command argument was not an integer") - ses.send("-ERR DELE command requires an integer argument") + s.logWarn("DELE command argument was not an integer") + s.send("-ERR DELE command requires an integer argument") return } if msgNum < 1 { - ses.logWarn("DELE command argument was less than 1") - ses.send("-ERR DELE argument must be greater than 0") + s.logWarn("DELE command argument was less than 1") + s.send("-ERR DELE argument must be greater than 0") return } - if int(msgNum) > len(ses.messages) { - ses.logWarn("DELE command argument was greater than number of messages") - ses.send("-ERR DELE argument must not exceed the number of messages") + if int(msgNum) > len(s.messages) { + s.logWarn("DELE command argument was greater than number of messages") + s.send("-ERR DELE argument must not exceed the number of messages") return } - if ses.retain[msgNum-1] { - ses.retain[msgNum-1] = false - ses.msgCount-- - ses.send(fmt.Sprintf("+OK Deleted message %v", msgNum)) + if s.retain[msgNum-1] { + s.retain[msgNum-1] = false + s.msgCount-- + s.send(fmt.Sprintf("+OK Deleted message %v", msgNum)) } else { - ses.logWarn("Client tried to DELE an already deleted message") - ses.send(fmt.Sprintf("-ERR Message %v has already been deleted", msgNum)) + s.logWarn("Client tried to DELE an already deleted message") + s.send(fmt.Sprintf("-ERR Message %v has already been deleted", msgNum)) } case "RETR": if len(args) != 1 { - ses.logWarn("RETR command had invalid number of arguments") - ses.send("-ERR RETR command requires a single argument") + s.logWarn("RETR command had invalid number of arguments") + s.send("-ERR RETR command requires a single argument") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - ses.logWarn("RETR command argument was not an integer") - ses.send("-ERR RETR command requires an integer argument") + s.logWarn("RETR command argument was not an integer") + s.send("-ERR RETR command requires an integer argument") return } if msgNum < 1 { - ses.logWarn("RETR command argument was less than 1") - ses.send("-ERR RETR argument must be greater than 0") + s.logWarn("RETR command argument was less than 1") + s.send("-ERR RETR argument must be greater than 0") return } - if int(msgNum) > len(ses.messages) { - ses.logWarn("RETR command argument was greater than number of messages") - ses.send("-ERR RETR argument must not exceed the number of messages") + if int(msgNum) > len(s.messages) { + s.logWarn("RETR command argument was greater than number of messages") + s.send("-ERR RETR argument must not exceed the number of messages") return } - ses.send(fmt.Sprintf("+OK %v bytes follows", ses.messages[msgNum-1].Size())) - ses.sendMessage(ses.messages[msgNum-1]) + s.send(fmt.Sprintf("+OK %v bytes follows", s.messages[msgNum-1].Size())) + s.sendMessage(s.messages[msgNum-1]) case "TOP": if len(args) != 2 { - ses.logWarn("TOP command had invalid number of arguments") - ses.send("-ERR TOP command requires two arguments") + s.logWarn("TOP command had invalid number of arguments") + s.send("-ERR TOP command requires two arguments") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - ses.logWarn("TOP command first argument was not an integer") - ses.send("-ERR TOP command requires an integer argument") + s.logWarn("TOP command first argument was not an integer") + s.send("-ERR TOP command requires an integer argument") return } if msgNum < 1 { - ses.logWarn("TOP command first argument was less than 1") - ses.send("-ERR TOP first argument must be greater than 0") + s.logWarn("TOP command first argument was less than 1") + s.send("-ERR TOP first argument must be greater than 0") return } - if int(msgNum) > len(ses.messages) { - ses.logWarn("TOP command first argument was greater than number of messages") - ses.send("-ERR TOP first argument must not exceed the number of messages") + if int(msgNum) > len(s.messages) { + s.logWarn("TOP command first argument was greater than number of messages") + s.send("-ERR TOP first argument must not exceed the number of messages") return } var lines int64 lines, err = strconv.ParseInt(args[1], 10, 32) if err != nil { - ses.logWarn("TOP command second argument was not an integer") - ses.send("-ERR TOP command requires an integer argument") + s.logWarn("TOP command second argument was not an integer") + s.send("-ERR TOP command requires an integer argument") return } if lines < 0 { - ses.logWarn("TOP command second argument was negative") - ses.send("-ERR TOP second argument must be non-negative") + s.logWarn("TOP command second argument was negative") + s.send("-ERR TOP second argument must be non-negative") return } - ses.send("+OK Top of message follows") - ses.sendMessageTop(ses.messages[msgNum-1], int(lines)) + s.send("+OK Top of message follows") + s.sendMessageTop(s.messages[msgNum-1], int(lines)) case "QUIT": - ses.send("+OK We will process your deletes") - ses.processDeletes() - ses.enterState(QUIT) + s.send("+OK We will process your deletes") + s.processDeletes() + s.enterState(QUIT) case "NOOP": - ses.send("+OK I have sucessfully done nothing") + s.send("+OK I have sucessfully done nothing") case "RSET": // Reset session, don't actually delete anything I told you to - ses.logTrace("Resetting session state on RSET request") - ses.reset() - ses.send("+OK Session reset") + s.logTrace("Resetting session state on RSET request") + s.reset() + s.send("+OK Session reset") default: - ses.ooSeq(cmd) + s.ooSeq(cmd) } } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg storage.Message) { +func (s *Session) sendMessage(msg storage.Message) { reader, err := msg.Source() if err != nil { - ses.logError("Failed to read message for RETR command") - ses.send("-ERR Failed to RETR that message, internal error") + s.logError("Failed to read message for RETR command") + s.send("-ERR Failed to RETR that message, internal error") return } defer func() { if err := reader.Close(); err != nil { - ses.logError("Failed to close message: %v", err) + s.logError("Failed to close message: %v", err) } }() @@ -435,29 +435,29 @@ func (ses *Session) sendMessage(msg storage.Message) { if strings.HasPrefix(line, ".") { line = "." + line } - ses.send(line) + s.send(line) } if err = scanner.Err(); err != nil { - ses.logError("Failed to read message for RETR command") - ses.send(".") - ses.send("-ERR Failed to RETR that message, internal error") + s.logError("Failed to read message for RETR command") + s.send(".") + s.send("-ERR Failed to RETR that message, internal error") return } - ses.send(".") + s.send(".") } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { +func (s *Session) sendMessageTop(msg storage.Message, lineCount int) { reader, err := msg.Source() if err != nil { - ses.logError("Failed to read message for RETR command") - ses.send("-ERR Failed to RETR that message, internal error") + s.logError("Failed to read message for RETR command") + s.send("-ERR Failed to RETR that message, internal error") return } defer func() { if err := reader.Close(); err != nil { - ses.logError("Failed to close message: %v", err) + s.logError("Failed to close message: %v", err) } }() @@ -482,85 +482,85 @@ func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { inBody = true } } - ses.send(line) + s.send(line) } if err = scanner.Err(); err != nil { - ses.logError("Failed to read message for RETR command") - ses.send(".") - ses.send("-ERR Failed to RETR that message, internal error") + s.logError("Failed to read message for RETR command") + s.send(".") + s.send("-ERR Failed to RETR that message, internal error") return } - ses.send(".") + s.send(".") } // Load the users mailbox -func (ses *Session) loadMailbox() { - m, err := ses.server.store.GetMessages(ses.user) +func (s *Session) loadMailbox() { + m, err := s.server.store.GetMessages(s.user) if err != nil { - ses.logError("Failed to load messages for %v: %v", ses.user, err) + s.logError("Failed to load messages for %v: %v", s.user, err) } - ses.messages = m - ses.retainAll() + s.messages = m + s.retainAll() } // Reset retain flag to true for all messages -func (ses *Session) retainAll() { - ses.retain = make([]bool, len(ses.messages)) - for i := range ses.retain { - ses.retain[i] = true +func (s *Session) retainAll() { + s.retain = make([]bool, len(s.messages)) + for i := range s.retain { + s.retain[i] = true } - ses.msgCount = len(ses.messages) + s.msgCount = len(s.messages) } // This would be considered the "UPDATE" state in the RFC, but it does not fit // with our state-machine design here, since no commands are accepted - it just // indicates that the session was closed cleanly and that deletes should be // processed. -func (ses *Session) processDeletes() { - ses.logInfo("Processing deletes") - for i, msg := range ses.messages { - if !ses.retain[i] { - ses.logTrace("Deleting %v", msg) - if err := ses.server.store.RemoveMessage(ses.user, msg.ID()); err != nil { - ses.logWarn("Error deleting %v: %v", msg, err) +func (s *Session) processDeletes() { + s.logInfo("Processing deletes") + for i, msg := range s.messages { + if !s.retain[i] { + s.logTrace("Deleting %v", msg) + if err := s.server.store.RemoveMessage(s.user, msg.ID()); err != nil { + s.logWarn("Error deleting %v: %v", msg, err) } } } } -func (ses *Session) enterState(state State) { - ses.state = state - ses.logTrace("Entering state %v", state) +func (s *Session) enterState(state State) { + s.state = state + s.logTrace("Entering state %v", state) } // Calculate the next read or write deadline based on maxIdleSeconds -func (ses *Session) nextDeadline() time.Time { - return time.Now().Add(ses.server.timeout) +func (s *Session) nextDeadline() time.Time { + return time.Now().Add(s.server.timeout) } // Send requested message, store errors in Session.sendError -func (ses *Session) send(msg string) { - if err := ses.conn.SetWriteDeadline(ses.nextDeadline()); err != nil { - ses.sendError = err +func (s *Session) send(msg string) { + if err := s.conn.SetWriteDeadline(s.nextDeadline()); err != nil { + s.sendError = err return } - if _, err := fmt.Fprint(ses.conn, msg+"\r\n"); err != nil { - ses.sendError = err - ses.logWarn("Failed to send: '%v'", msg) + if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { + s.sendError = err + s.logWarn("Failed to send: '%v'", msg) return } - ses.logTrace(">> %v >>", msg) + s.logTrace(">> %v >>", msg) } // readByteLine reads a line of input into the provided buffer. Does // not reset the Buffer - please do so prior to calling. -func (ses *Session) readByteLine(buf *bytes.Buffer) error { - if err := ses.conn.SetReadDeadline(ses.nextDeadline()); err != nil { +func (s *Session) readByteLine(buf *bytes.Buffer) error { + if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return err } for { - line, err := ses.reader.ReadBytes('\r') + line, err := s.reader.ReadBytes('\r') if err != nil { return err } @@ -568,7 +568,7 @@ func (ses *Session) readByteLine(buf *bytes.Buffer) error { return err } // Read the next byte looking for '\n' - c, err := ses.reader.ReadByte() + c, err := s.reader.ReadByte() if err != nil { return err } @@ -585,19 +585,19 @@ func (ses *Session) readByteLine(buf *bytes.Buffer) error { } // Reads a line of input -func (ses *Session) readLine() (line string, err error) { - if err = ses.conn.SetReadDeadline(ses.nextDeadline()); err != nil { +func (s *Session) readLine() (line string, err error) { + if err = s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return "", err } - line, err = ses.reader.ReadString('\n') + line, err = s.reader.ReadString('\n') if err != nil { return "", err } - ses.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) + s.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) return line, nil } -func (ses *Session) parseCmd(line string) (cmd string, args []string, ok bool) { +func (s *Session) parseCmd(line string) (cmd string, args []string, ok bool) { line = strings.TrimRight(line, "\r\n") if line == "" { return "", nil, true @@ -607,32 +607,32 @@ func (ses *Session) parseCmd(line string) (cmd string, args []string, ok bool) { return strings.ToUpper(words[0]), words[1:], true } -func (ses *Session) reset() { - ses.retainAll() +func (s *Session) reset() { + s.retainAll() } -func (ses *Session) ooSeq(cmd string) { - ses.send(fmt.Sprintf("-ERR Command %v is out of sequence", cmd)) - ses.logWarn("Wasn't expecting %v here", cmd) +func (s *Session) ooSeq(cmd string) { + s.send(fmt.Sprintf("-ERR Command %v is out of sequence", cmd)) + s.logWarn("Wasn't expecting %v here", cmd) } // Session specific logging methods -func (ses *Session) logTrace(msg string, args ...interface{}) { - log.Tracef("POP3[%v]<%v> %v", ses.remoteHost, ses.id, fmt.Sprintf(msg, args...)) +func (s *Session) logTrace(msg string, args ...interface{}) { + log.Tracef("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ses *Session) logInfo(msg string, args ...interface{}) { - log.Infof("POP3[%v]<%v> %v", ses.remoteHost, ses.id, fmt.Sprintf(msg, args...)) +func (s *Session) logInfo(msg string, args ...interface{}) { + log.Infof("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ses *Session) logWarn(msg string, args ...interface{}) { +func (s *Session) logWarn(msg string, args ...interface{}) { // Update metrics //expWarnsTotal.Add(1) - log.Warnf("POP3[%v]<%v> %v", ses.remoteHost, ses.id, fmt.Sprintf(msg, args...)) + log.Warnf("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ses *Session) logError(msg string, args ...interface{}) { +func (s *Session) logError(msg string, args ...interface{}) { // Update metrics //expErrorsTotal.Add(1) - log.Errorf("POP3[%v]<%v> %v", ses.remoteHost, ses.id, fmt.Sprintf(msg, args...)) + log.Errorf("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 634c56b..0a006b3 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -96,8 +96,8 @@ func NewSession(server *Server, id int, conn net.Conn) *Session { } } -func (ss *Session) String() string { - return fmt.Sprintf("Session{id: %v, state: %v}", ss.id, ss.state) +func (s *Session) String() string { + return fmt.Sprintf("Session{id: %v, state: %v}", s.id, s.state) } /* Session flow: @@ -118,27 +118,27 @@ func (s *Server) startSession(id int, conn net.Conn) { expConnectsCurrent.Add(-1) }() - ss := NewSession(s, id, conn) - ss.greet() + ssn := NewSession(s, id, conn) + ssn.greet() // This is our command reading loop - for ss.state != QUIT && ss.sendError == nil { - if ss.state == DATA { + for ssn.state != QUIT && ssn.sendError == nil { + if ssn.state == DATA { // Special case, does not use SMTP command format - ss.dataHandler() + ssn.dataHandler() continue } - line, err := ss.readLine() + line, err := ssn.readLine() if err == nil { - if cmd, arg, ok := ss.parseCmd(line); ok { + if cmd, arg, ok := ssn.parseCmd(line); ok { // Check against valid SMTP commands if cmd == "" { - ss.send("500 Speak up") + ssn.send("500 Speak up") continue } if !commands[cmd] { - ss.send(fmt.Sprintf("500 Syntax error, %v command unrecognized", cmd)) - ss.logWarn("Unrecognized command: %v", cmd) + ssn.send(fmt.Sprintf("500 Syntax error, %v command unrecognized", cmd)) + ssn.logWarn("Unrecognized command: %v", cmd) continue } @@ -146,99 +146,99 @@ func (s *Server) startSession(id int, conn net.Conn) { switch cmd { case "SEND", "SOML", "SAML", "EXPN", "HELP", "TURN": // These commands are not implemented in any state - ss.send(fmt.Sprintf("502 %v command not implemented", cmd)) - ss.logWarn("Command %v not implemented by Inbucket", cmd) + ssn.send(fmt.Sprintf("502 %v command not implemented", cmd)) + ssn.logWarn("Command %v not implemented by Inbucket", cmd) continue case "VRFY": - ss.send("252 Cannot VRFY user, but will accept message") + ssn.send("252 Cannot VRFY user, but will accept message") continue case "NOOP": - ss.send("250 I have sucessfully done nothing") + ssn.send("250 I have sucessfully done nothing") continue case "RSET": // Reset session - ss.logTrace("Resetting session state on RSET request") - ss.reset() - ss.send("250 Session reset") + ssn.logTrace("Resetting session state on RSET request") + ssn.reset() + ssn.send("250 Session reset") continue case "QUIT": - ss.send("221 Goodnight and good luck") - ss.enterState(QUIT) + ssn.send("221 Goodnight and good luck") + ssn.enterState(QUIT) continue } // Send command to handler for current state - switch ss.state { + switch ssn.state { case GREET: - ss.greetHandler(cmd, arg) + ssn.greetHandler(cmd, arg) continue case READY: - ss.readyHandler(cmd, arg) + ssn.readyHandler(cmd, arg) continue case MAIL: - ss.mailHandler(cmd, arg) + ssn.mailHandler(cmd, arg) continue } - ss.logError("Session entered unexpected state %v", ss.state) + ssn.logError("Session entered unexpected state %v", ssn.state) break } else { - ss.send("500 Syntax error, command garbled") + ssn.send("500 Syntax error, command garbled") } } else { // readLine() returned an error if err == io.EOF { - switch ss.state { + switch ssn.state { case GREET, READY: // EOF is common here - ss.logInfo("Client closed connection (state %v)", ss.state) + ssn.logInfo("Client closed connection (state %v)", ssn.state) default: - ss.logWarn("Got EOF while in state %v", ss.state) + ssn.logWarn("Got EOF while in state %v", ssn.state) } break } // not an EOF - ss.logWarn("Connection error: %v", err) + ssn.logWarn("Connection error: %v", err) if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { - ss.send("221 Idle timeout, bye bye") + ssn.send("221 Idle timeout, bye bye") break } } - ss.send("221 Connection error, sorry") + ssn.send("221 Connection error, sorry") break } } - if ss.sendError != nil { - ss.logWarn("Network send error: %v", ss.sendError) + if ssn.sendError != nil { + ssn.logWarn("Network send error: %v", ssn.sendError) } - ss.logInfo("Closing connection") + ssn.logInfo("Closing connection") } // GREET state -> waiting for HELO -func (ss *Session) greetHandler(cmd string, arg string) { +func (s *Session) greetHandler(cmd string, arg string) { switch cmd { case "HELO": domain, err := parseHelloArgument(arg) if err != nil { - ss.send("501 Domain/address argument required for HELO") + s.send("501 Domain/address argument required for HELO") return } - ss.remoteDomain = domain - ss.send("250 Great, let's get this show on the road") - ss.enterState(READY) + s.remoteDomain = domain + s.send("250 Great, let's get this show on the road") + s.enterState(READY) case "EHLO": domain, err := parseHelloArgument(arg) if err != nil { - ss.send("501 Domain/address argument required for EHLO") + s.send("501 Domain/address argument required for EHLO") return } - ss.remoteDomain = domain - ss.send("250-Great, let's get this show on the road") - ss.send("250-8BITMIME") - ss.send(fmt.Sprintf("250 SIZE %v", ss.server.maxMessageBytes)) - ss.enterState(READY) + s.remoteDomain = domain + s.send("250-Great, let's get this show on the road") + s.send("250-8BITMIME") + s.send(fmt.Sprintf("250 SIZE %v", s.server.maxMessageBytes)) + s.enterState(READY) default: - ss.ooSeq(cmd) + s.ooSeq(cmd) } } @@ -254,139 +254,139 @@ func parseHelloArgument(arg string) (string, error) { } // READY state -> waiting for MAIL -func (ss *Session) readyHandler(cmd string, arg string) { +func (s *Session) readyHandler(cmd string, arg string) { if cmd == "MAIL" { // Match FROM, while accepting '>' as quoted pair and in double quoted strings // (?i) makes the regex case insensitive, (?:) is non-grouping sub-match re := regexp.MustCompile("(?i)^FROM:\\s*<((?:\\\\>|[^>])+|\"[^\"]+\"@[^>]+)>( [\\w= ]+)?$") m := re.FindStringSubmatch(arg) if m == nil { - ss.send("501 Was expecting MAIL arg syntax of FROM:
") - ss.logWarn("Bad MAIL argument: %q", arg) + s.send("501 Was expecting MAIL arg syntax of FROM:
") + s.logWarn("Bad MAIL argument: %q", arg) return } from := m[1] if _, _, err := policy.ParseEmailAddress(from); err != nil { - ss.send("501 Bad sender address syntax") - ss.logWarn("Bad address as MAIL arg: %q, %s", from, err) + s.send("501 Bad sender address syntax") + s.logWarn("Bad address as MAIL arg: %q, %s", from, err) return } // This is where the client may put BODY=8BITMIME, but we already // read the DATA as bytes, so it does not effect our processing. if m[2] != "" { - args, ok := ss.parseArgs(m[2]) + args, ok := s.parseArgs(m[2]) if !ok { - ss.send("501 Unable to parse MAIL ESMTP parameters") - ss.logWarn("Bad MAIL argument: %q", arg) + s.send("501 Unable to parse MAIL ESMTP parameters") + s.logWarn("Bad MAIL argument: %q", arg) return } if args["SIZE"] != "" { size, err := strconv.ParseInt(args["SIZE"], 10, 32) if err != nil { - ss.send("501 Unable to parse SIZE as an integer") - ss.logWarn("Unable to parse SIZE %q as an integer", args["SIZE"]) + s.send("501 Unable to parse SIZE as an integer") + s.logWarn("Unable to parse SIZE %q as an integer", args["SIZE"]) return } - if int(size) > ss.server.maxMessageBytes { - ss.send("552 Max message size exceeded") - ss.logWarn("Client wanted to send oversized message: %v", args["SIZE"]) + if int(size) > s.server.maxMessageBytes { + s.send("552 Max message size exceeded") + s.logWarn("Client wanted to send oversized message: %v", args["SIZE"]) return } } } - ss.from = from - ss.logInfo("Mail from: %v", from) - ss.send(fmt.Sprintf("250 Roger, accepting mail from <%v>", from)) - ss.enterState(MAIL) + s.from = from + s.logInfo("Mail from: %v", from) + s.send(fmt.Sprintf("250 Roger, accepting mail from <%v>", from)) + s.enterState(MAIL) } else { - ss.ooSeq(cmd) + s.ooSeq(cmd) } } // MAIL state -> waiting for RCPTs followed by DATA -func (ss *Session) mailHandler(cmd string, arg string) { +func (s *Session) mailHandler(cmd string, arg string) { switch cmd { case "RCPT": if (len(arg) < 4) || (strings.ToUpper(arg[0:3]) != "TO:") { - ss.send("501 Was expecting RCPT arg syntax of TO:
") - ss.logWarn("Bad RCPT argument: %q", arg) + s.send("501 Was expecting RCPT arg syntax of TO:
") + s.logWarn("Bad RCPT argument: %q", arg) return } // This trim is probably too forgiving addr := strings.Trim(arg[3:], "<> ") - recip, err := ss.server.apolicy.NewRecipient(addr) + recip, err := s.server.apolicy.NewRecipient(addr) if err != nil { - ss.send("501 Bad recipient address syntax") - ss.logWarn("Bad address as RCPT arg: %q, %s", addr, err) + s.send("501 Bad recipient address syntax") + s.logWarn("Bad address as RCPT arg: %q, %s", addr, err) return } - if len(ss.recipients) >= ss.server.maxRecips { - ss.logWarn("Maximum limit of %v recipients reached", ss.server.maxRecips) - ss.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", ss.server.maxRecips)) + if len(s.recipients) >= s.server.maxRecips { + s.logWarn("Maximum limit of %v recipients reached", s.server.maxRecips) + s.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", s.server.maxRecips)) return } - ss.recipients = append(ss.recipients, recip) - ss.logInfo("Recipient: %v", addr) - ss.send(fmt.Sprintf("250 I'll make sure <%v> gets this", addr)) + s.recipients = append(s.recipients, recip) + s.logInfo("Recipient: %v", addr) + s.send(fmt.Sprintf("250 I'll make sure <%v> gets this", addr)) return case "DATA": if arg != "" { - ss.send("501 DATA command should not have any arguments") - ss.logWarn("Got unexpected args on DATA: %q", arg) + s.send("501 DATA command should not have any arguments") + s.logWarn("Got unexpected args on DATA: %q", arg) return } - if len(ss.recipients) > 0 { + if len(s.recipients) > 0 { // We have recipients, go to accept data - ss.enterState(DATA) + s.enterState(DATA) return } // DATA out of sequence - ss.ooSeq(cmd) + s.ooSeq(cmd) return } - ss.ooSeq(cmd) + s.ooSeq(cmd) } // DATA -func (ss *Session) dataHandler() { - ss.send("354 Start mail input; end with .") +func (s *Session) dataHandler() { + s.send("354 Start mail input; end with .") msgBuf := &bytes.Buffer{} for { - lineBuf, err := ss.readByteLine() + lineBuf, err := s.readByteLine() if err != nil { if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { - ss.send("221 Idle timeout, bye bye") + s.send("221 Idle timeout, bye bye") } } - ss.logWarn("Error: %v while reading", err) - ss.enterState(QUIT) + s.logWarn("Error: %v while reading", err) + s.enterState(QUIT) return } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete. tstamp := time.Now().Format(timeStampFormat) - for _, recip := range ss.recipients { + for _, recip := range s.recipients { if recip.ShouldStore() { // Generate Received header. prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address.Address, + s.remoteDomain, s.remoteHost, s.server.domain, recip.Address.Address, tstamp) // Deliver message. - _, err := ss.server.manager.Deliver( - recip, ss.from, ss.recipients, prefix, msgBuf.Bytes()) + _, err := s.server.manager.Deliver( + recip, s.from, s.recipients, prefix, msgBuf.Bytes()) if err != nil { - ss.logError("delivery for %v: %v", recip.LocalPart, err) - ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) - ss.reset() + s.logError("delivery for %v: %v", recip.LocalPart, err) + s.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) + s.reset() return } } expReceivedTotal.Add(1) } - ss.send("250 Mail accepted for delivery") - ss.logInfo("Message size %v bytes", msgBuf.Len()) - ss.reset() + s.send("250 Mail accepted for delivery") + s.logInfo("Message size %v bytes", msgBuf.Len()) + s.reset() return } // RFC: remove leading periods from DATA. @@ -394,84 +394,84 @@ func (ss *Session) dataHandler() { lineBuf = lineBuf[1:] } msgBuf.Write(lineBuf) - if msgBuf.Len() > ss.server.maxMessageBytes { - ss.send("552 Maximum message size exceeded") - ss.logWarn("Max message size exceeded while in DATA") - ss.reset() + if msgBuf.Len() > s.server.maxMessageBytes { + s.send("552 Maximum message size exceeded") + s.logWarn("Max message size exceeded while in DATA") + s.reset() return } } } -func (ss *Session) enterState(state State) { - ss.state = state - ss.logTrace("Entering state %v", state) +func (s *Session) enterState(state State) { + s.state = state + s.logTrace("Entering state %v", state) } -func (ss *Session) greet() { - ss.send(fmt.Sprintf("220 %v Inbucket SMTP ready", ss.server.domain)) +func (s *Session) greet() { + s.send(fmt.Sprintf("220 %v Inbucket SMTP ready", s.server.domain)) } // Calculate the next read or write deadline based on maxIdle -func (ss *Session) nextDeadline() time.Time { - return time.Now().Add(ss.server.timeout) +func (s *Session) nextDeadline() time.Time { + return time.Now().Add(s.server.timeout) } // Send requested message, store errors in Session.sendError -func (ss *Session) send(msg string) { - if err := ss.conn.SetWriteDeadline(ss.nextDeadline()); err != nil { - ss.sendError = err +func (s *Session) send(msg string) { + if err := s.conn.SetWriteDeadline(s.nextDeadline()); err != nil { + s.sendError = err return } - if _, err := fmt.Fprint(ss.conn, msg+"\r\n"); err != nil { - ss.sendError = err - ss.logWarn("Failed to send: %q", msg) + if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { + s.sendError = err + s.logWarn("Failed to send: %q", msg) return } - ss.logTrace(">> %v >>", msg) + s.logTrace(">> %v >>", msg) } // readByteLine reads a line of input, returns byte slice. -func (ss *Session) readByteLine() ([]byte, error) { - if err := ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { +func (s *Session) readByteLine() ([]byte, error) { + if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return nil, err } - return ss.reader.ReadBytes('\n') + return s.reader.ReadBytes('\n') } // Reads a line of input -func (ss *Session) readLine() (line string, err error) { - if err = ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { +func (s *Session) readLine() (line string, err error) { + if err = s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return "", err } - line, err = ss.reader.ReadString('\n') + line, err = s.reader.ReadString('\n') if err != nil { return "", err } - ss.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) + s.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) return line, nil } -func (ss *Session) parseCmd(line string) (cmd string, arg string, ok bool) { +func (s *Session) parseCmd(line string) (cmd string, arg string, ok bool) { line = strings.TrimRight(line, "\r\n") l := len(line) switch { case l == 0: return "", "", true case l < 4: - ss.logWarn("Command too short: %q", line) + s.logWarn("Command too short: %q", line) return "", "", false case l == 4: return strings.ToUpper(line), "", true case l == 5: // Too long to be only command, too short to have args - ss.logWarn("Mangled command: %q", line) + s.logWarn("Mangled command: %q", line) return "", "", false } // If we made it here, command is long enough to have args if line[4] != ' ' { // There wasn't a space after the command? - ss.logWarn("Mangled command: %q", line) + s.logWarn("Mangled command: %q", line) return "", "", false } // I'm not sure if we should trim the args or not, but we will for now @@ -483,49 +483,49 @@ func (ss *Session) parseCmd(line string) (cmd string, arg string, ok bool) { // string: // " BODY=8BITMIME SIZE=1024" // The leading space is mandatory. -func (ss *Session) parseArgs(arg string) (args map[string]string, ok bool) { +func (s *Session) parseArgs(arg string) (args map[string]string, ok bool) { args = make(map[string]string) re := regexp.MustCompile(` (\w+)=(\w+)`) pm := re.FindAllStringSubmatch(arg, -1) if pm == nil { - ss.logWarn("Failed to parse arg string: %q") + s.logWarn("Failed to parse arg string: %q") return nil, false } for _, m := range pm { args[strings.ToUpper(m[1])] = m[2] } - ss.logTrace("ESMTP params: %v", args) + s.logTrace("ESMTP params: %v", args) return args, true } -func (ss *Session) reset() { - ss.enterState(READY) - ss.from = "" - ss.recipients = nil +func (s *Session) reset() { + s.enterState(READY) + s.from = "" + s.recipients = nil } -func (ss *Session) ooSeq(cmd string) { - ss.send(fmt.Sprintf("503 Command %v is out of sequence", cmd)) - ss.logWarn("Wasn't expecting %v here", cmd) +func (s *Session) ooSeq(cmd string) { + s.send(fmt.Sprintf("503 Command %v is out of sequence", cmd)) + s.logWarn("Wasn't expecting %v here", cmd) } // Session specific logging methods -func (ss *Session) logTrace(msg string, args ...interface{}) { - log.Tracef("SMTP[%v]<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...)) +func (s *Session) logTrace(msg string, args ...interface{}) { + log.Tracef("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ss *Session) logInfo(msg string, args ...interface{}) { - log.Infof("SMTP[%v]<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...)) +func (s *Session) logInfo(msg string, args ...interface{}) { + log.Infof("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ss *Session) logWarn(msg string, args ...interface{}) { +func (s *Session) logWarn(msg string, args ...interface{}) { // Update metrics expWarnsTotal.Add(1) - log.Warnf("SMTP[%v]<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...)) + log.Warnf("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } -func (ss *Session) logError(msg string, args ...interface{}) { +func (s *Session) logError(msg string, args ...interface{}) { // Update metrics expErrorsTotal.Add(1) - log.Errorf("SMTP[%v]<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...)) + log.Errorf("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) } From 779b1e63afd4b4b66139e9f4322e3a08e878b5e2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 27 Mar 2018 21:52:28 -0700 Subject: [PATCH 52/82] smtp, pop3: Use zerolog for session logging #90 --- pkg/server/pop3/handler.go | 140 ++++++++++++++++--------------------- pkg/server/smtp/handler.go | 104 ++++++++++++--------------- 2 files changed, 105 insertions(+), 139 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 61384d7..8e8eeb8 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -11,8 +11,9 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) // State tracks the current mode of our POP3 state machine @@ -68,14 +69,15 @@ type Session struct { messages []storage.Message // Slice of messages in mailbox retain []bool // Messages to retain upon UPDATE (true=retain) msgCount int // Number of undeleted messages + logger zerolog.Logger } // NewSession creates a new POP3 session -func NewSession(server *Server, id int, conn net.Conn) *Session { +func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) return &Session{server: server, id: id, conn: conn, state: AUTHORIZATION, - reader: reader, remoteHost: host} + reader: reader, remoteHost: host, logger: logger} } func (s *Session) String() string { @@ -90,17 +92,17 @@ func (s *Session) String() string { * 5. Goto 2 */ func (s *Server) startSession(id int, conn net.Conn) { - log.Infof("POP3 connection from %v, starting session <%v>", conn.RemoteAddr(), id) - //expConnectsCurrent.Add(1) + logger := log.With().Str("module", "pop3").Str("remote", conn.RemoteAddr().String()). + Int("session", id).Logger() + logger.Info().Msg("Starting POP3 session") defer func() { if err := conn.Close(); err != nil { - log.Errorf("Error closing POP3 connection for <%v>: %v", id, err) + logger.Warn().Err(err).Msg("Closing connection") } s.waitgroup.Done() - //expConnectsCurrent.Add(-1) }() - ssn := NewSession(s, id, conn) + ssn := NewSession(s, id, conn, logger) ssn.send(fmt.Sprintf("+OK Inbucket POP3 server ready <%v.%v@%v>", os.Getpid(), time.Now().Unix(), s.domain)) @@ -116,7 +118,7 @@ func (s *Server) startSession(id int, conn net.Conn) { } if !commands[cmd] { ssn.send(fmt.Sprintf("-ERR Syntax error, %v command unrecognized", cmd)) - ssn.logWarn("Unrecognized command: %v", cmd) + ssn.logger.Warn().Msgf("Unrecognized command: %v", cmd) continue } @@ -142,7 +144,7 @@ func (s *Server) startSession(id int, conn net.Conn) { ssn.transactionHandler(cmd, arg) continue } - ssn.logError("Session entered unexpected state %v", ssn.state) + ssn.logger.Error().Msgf("Session entered unexpected state %v", ssn.state) break } else { ssn.send("-ERR Syntax error, command garbled") @@ -153,14 +155,14 @@ func (s *Server) startSession(id int, conn net.Conn) { switch ssn.state { case AUTHORIZATION: // EOF is common here - ssn.logInfo("Client closed connection (state %v)", ssn.state) + ssn.logger.Info().Msgf("Client closed connection (state %v)", ssn.state) default: - ssn.logWarn("Got EOF while in state %v", ssn.state) + ssn.logger.Warn().Msgf("Got EOF while in state %v", ssn.state) } break } // not an EOF - ssn.logWarn("Connection error: %v", err) + ssn.logger.Warn().Msgf("Connection error: %v", err) if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { ssn.send("-ERR Idle timeout, bye bye") @@ -172,9 +174,9 @@ func (s *Server) startSession(id int, conn net.Conn) { } } if ssn.sendError != nil { - ssn.logWarn("Network send error: %v", ssn.sendError) + ssn.logger.Warn().Msgf("Network send error: %v", ssn.sendError) } - ssn.logInfo("Closing connection") + ssn.logger.Info().Msgf("Closing connection") } // AUTHORIZATION state @@ -200,7 +202,7 @@ func (s *Session) authorizationHandler(cmd string, args []string) { } case "APOP": if len(args) != 2 { - s.logWarn("Expected two arguments for APOP") + s.logger.Warn().Msgf("Expected two arguments for APOP") s.send("-ERR APOP requires two arguments") return } @@ -218,7 +220,7 @@ func (s *Session) transactionHandler(cmd string, args []string) { switch cmd { case "STAT": if len(args) != 0 { - s.logWarn("STAT got an unexpected argument") + s.logger.Warn().Msgf("STAT got an unexpected argument") s.send("-ERR STAT command must have no arguments") return } @@ -233,29 +235,29 @@ func (s *Session) transactionHandler(cmd string, args []string) { s.send(fmt.Sprintf("+OK %v %v", count, size)) case "LIST": if len(args) > 1 { - s.logWarn("LIST command had more than 1 argument") + s.logger.Warn().Msgf("LIST command had more than 1 argument") s.send("-ERR LIST command must have zero or one argument") return } if len(args) == 1 { msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - s.logWarn("LIST command argument was not an integer") + s.logger.Warn().Msgf("LIST command argument was not an integer") s.send("-ERR LIST command requires an integer argument") return } if msgNum < 1 { - s.logWarn("LIST command argument was less than 1") + s.logger.Warn().Msgf("LIST command argument was less than 1") s.send("-ERR LIST argument must be greater than 0") return } if int(msgNum) > len(s.messages) { - s.logWarn("LIST command argument was greater than number of messages") + s.logger.Warn().Msgf("LIST command argument was greater than number of messages") s.send("-ERR LIST argument must not exceed the number of messages") return } if !s.retain[msgNum-1] { - s.logWarn("Client tried to LIST a message it had deleted") + s.logger.Warn().Msgf("Client tried to LIST a message it had deleted") s.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) return } @@ -271,29 +273,29 @@ func (s *Session) transactionHandler(cmd string, args []string) { } case "UIDL": if len(args) > 1 { - s.logWarn("UIDL command had more than 1 argument") + s.logger.Warn().Msgf("UIDL command had more than 1 argument") s.send("-ERR UIDL command must have zero or one argument") return } if len(args) == 1 { msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - s.logWarn("UIDL command argument was not an integer") + s.logger.Warn().Msgf("UIDL command argument was not an integer") s.send("-ERR UIDL command requires an integer argument") return } if msgNum < 1 { - s.logWarn("UIDL command argument was less than 1") + s.logger.Warn().Msgf("UIDL command argument was less than 1") s.send("-ERR UIDL argument must be greater than 0") return } if int(msgNum) > len(s.messages) { - s.logWarn("UIDL command argument was greater than number of messages") + s.logger.Warn().Msgf("UIDL command argument was greater than number of messages") s.send("-ERR UIDL argument must not exceed the number of messages") return } if !s.retain[msgNum-1] { - s.logWarn("Client tried to UIDL a message it had deleted") + s.logger.Warn().Msgf("Client tried to UIDL a message it had deleted") s.send(fmt.Sprintf("-ERR You deleted message %v", msgNum)) return } @@ -309,23 +311,23 @@ func (s *Session) transactionHandler(cmd string, args []string) { } case "DELE": if len(args) != 1 { - s.logWarn("DELE command had invalid number of arguments") + s.logger.Warn().Msgf("DELE command had invalid number of arguments") s.send("-ERR DELE command requires a single argument") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - s.logWarn("DELE command argument was not an integer") + s.logger.Warn().Msgf("DELE command argument was not an integer") s.send("-ERR DELE command requires an integer argument") return } if msgNum < 1 { - s.logWarn("DELE command argument was less than 1") + s.logger.Warn().Msgf("DELE command argument was less than 1") s.send("-ERR DELE argument must be greater than 0") return } if int(msgNum) > len(s.messages) { - s.logWarn("DELE command argument was greater than number of messages") + s.logger.Warn().Msgf("DELE command argument was greater than number of messages") s.send("-ERR DELE argument must not exceed the number of messages") return } @@ -334,28 +336,28 @@ func (s *Session) transactionHandler(cmd string, args []string) { s.msgCount-- s.send(fmt.Sprintf("+OK Deleted message %v", msgNum)) } else { - s.logWarn("Client tried to DELE an already deleted message") + s.logger.Warn().Msgf("Client tried to DELE an already deleted message") s.send(fmt.Sprintf("-ERR Message %v has already been deleted", msgNum)) } case "RETR": if len(args) != 1 { - s.logWarn("RETR command had invalid number of arguments") + s.logger.Warn().Msgf("RETR command had invalid number of arguments") s.send("-ERR RETR command requires a single argument") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - s.logWarn("RETR command argument was not an integer") + s.logger.Warn().Msgf("RETR command argument was not an integer") s.send("-ERR RETR command requires an integer argument") return } if msgNum < 1 { - s.logWarn("RETR command argument was less than 1") + s.logger.Warn().Msgf("RETR command argument was less than 1") s.send("-ERR RETR argument must be greater than 0") return } if int(msgNum) > len(s.messages) { - s.logWarn("RETR command argument was greater than number of messages") + s.logger.Warn().Msgf("RETR command argument was greater than number of messages") s.send("-ERR RETR argument must not exceed the number of messages") return } @@ -363,23 +365,23 @@ func (s *Session) transactionHandler(cmd string, args []string) { s.sendMessage(s.messages[msgNum-1]) case "TOP": if len(args) != 2 { - s.logWarn("TOP command had invalid number of arguments") + s.logger.Warn().Msgf("TOP command had invalid number of arguments") s.send("-ERR TOP command requires two arguments") return } msgNum, err := strconv.ParseInt(args[0], 10, 32) if err != nil { - s.logWarn("TOP command first argument was not an integer") + s.logger.Warn().Msgf("TOP command first argument was not an integer") s.send("-ERR TOP command requires an integer argument") return } if msgNum < 1 { - s.logWarn("TOP command first argument was less than 1") + s.logger.Warn().Msgf("TOP command first argument was less than 1") s.send("-ERR TOP first argument must be greater than 0") return } if int(msgNum) > len(s.messages) { - s.logWarn("TOP command first argument was greater than number of messages") + s.logger.Warn().Msgf("TOP command first argument was greater than number of messages") s.send("-ERR TOP first argument must not exceed the number of messages") return } @@ -387,12 +389,12 @@ func (s *Session) transactionHandler(cmd string, args []string) { var lines int64 lines, err = strconv.ParseInt(args[1], 10, 32) if err != nil { - s.logWarn("TOP command second argument was not an integer") + s.logger.Warn().Msgf("TOP command second argument was not an integer") s.send("-ERR TOP command requires an integer argument") return } if lines < 0 { - s.logWarn("TOP command second argument was negative") + s.logger.Warn().Msgf("TOP command second argument was negative") s.send("-ERR TOP second argument must be non-negative") return } @@ -406,7 +408,7 @@ func (s *Session) transactionHandler(cmd string, args []string) { s.send("+OK I have sucessfully done nothing") case "RSET": // Reset session, don't actually delete anything I told you to - s.logTrace("Resetting session state on RSET request") + s.logger.Debug().Msgf("Resetting session state on RSET request") s.reset() s.send("+OK Session reset") default: @@ -418,13 +420,13 @@ func (s *Session) transactionHandler(cmd string, args []string) { func (s *Session) sendMessage(msg storage.Message) { reader, err := msg.Source() if err != nil { - s.logError("Failed to read message for RETR command") + s.logger.Error().Msgf("Failed to read message for RETR command") s.send("-ERR Failed to RETR that message, internal error") return } defer func() { if err := reader.Close(); err != nil { - s.logError("Failed to close message: %v", err) + s.logger.Error().Msgf("Failed to close message: %v", err) } }() @@ -439,7 +441,7 @@ func (s *Session) sendMessage(msg storage.Message) { } if err = scanner.Err(); err != nil { - s.logError("Failed to read message for RETR command") + s.logger.Error().Msgf("Failed to read message for RETR command") s.send(".") s.send("-ERR Failed to RETR that message, internal error") return @@ -451,13 +453,13 @@ func (s *Session) sendMessage(msg storage.Message) { func (s *Session) sendMessageTop(msg storage.Message, lineCount int) { reader, err := msg.Source() if err != nil { - s.logError("Failed to read message for RETR command") + s.logger.Error().Msgf("Failed to read message for RETR command") s.send("-ERR Failed to RETR that message, internal error") return } defer func() { if err := reader.Close(); err != nil { - s.logError("Failed to close message: %v", err) + s.logger.Error().Msgf("Failed to close message: %v", err) } }() @@ -486,7 +488,7 @@ func (s *Session) sendMessageTop(msg storage.Message, lineCount int) { } if err = scanner.Err(); err != nil { - s.logError("Failed to read message for RETR command") + s.logger.Error().Msgf("Failed to read message for RETR command") s.send(".") s.send("-ERR Failed to RETR that message, internal error") return @@ -496,9 +498,10 @@ func (s *Session) sendMessageTop(msg storage.Message, lineCount int) { // Load the users mailbox func (s *Session) loadMailbox() { + s.logger = s.logger.With().Str("mailbox", s.user).Logger() m, err := s.server.store.GetMessages(s.user) if err != nil { - s.logError("Failed to load messages for %v: %v", s.user, err) + s.logger.Error().Msgf("Failed to load messages for %v: %v", s.user, err) } s.messages = m s.retainAll() @@ -518,12 +521,12 @@ func (s *Session) retainAll() { // indicates that the session was closed cleanly and that deletes should be // processed. func (s *Session) processDeletes() { - s.logInfo("Processing deletes") + s.logger.Info().Msgf("Processing deletes") for i, msg := range s.messages { if !s.retain[i] { - s.logTrace("Deleting %v", msg) + s.logger.Debug().Str("id", msg.ID()).Msg("Deleting message") if err := s.server.store.RemoveMessage(s.user, msg.ID()); err != nil { - s.logWarn("Error deleting %v: %v", msg, err) + s.logger.Warn().Str("id", msg.ID()).Err(err).Msg("Error deleting message") } } } @@ -531,7 +534,7 @@ func (s *Session) processDeletes() { func (s *Session) enterState(state State) { s.state = state - s.logTrace("Entering state %v", state) + s.logger.Debug().Msgf("Entering state %v", state) } // Calculate the next read or write deadline based on maxIdleSeconds @@ -547,10 +550,10 @@ func (s *Session) send(msg string) { } if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { s.sendError = err - s.logWarn("Failed to send: '%v'", msg) + s.logger.Warn().Msgf("Failed to send: '%v'", msg) return } - s.logTrace(">> %v >>", msg) + s.logger.Debug().Msgf(">> %v >>", msg) } // readByteLine reads a line of input into the provided buffer. Does @@ -593,7 +596,7 @@ func (s *Session) readLine() (line string, err error) { if err != nil { return "", err } - s.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) + s.logger.Debug().Msgf("<< %v <<", strings.TrimRight(line, "\r\n")) return line, nil } @@ -613,26 +616,5 @@ func (s *Session) reset() { func (s *Session) ooSeq(cmd string) { s.send(fmt.Sprintf("-ERR Command %v is out of sequence", cmd)) - s.logWarn("Wasn't expecting %v here", cmd) -} - -// Session specific logging methods -func (s *Session) logTrace(msg string, args ...interface{}) { - log.Tracef("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logInfo(msg string, args ...interface{}) { - log.Infof("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logWarn(msg string, args ...interface{}) { - // Update metrics - //expWarnsTotal.Add(1) - log.Warnf("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logError(msg string, args ...interface{}) { - // Update metrics - //expErrorsTotal.Add(1) - log.Errorf("POP3[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) + s.logger.Warn().Msgf("Wasn't expecting %v here", cmd) } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 0a006b3..d696776 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -11,8 +11,9 @@ import ( "strings" "time" - "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/policy" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) // State tracks the current mode of our SMTP state machine @@ -79,10 +80,11 @@ type Session struct { reader *bufio.Reader from string recipients []*policy.Recipient + logger zerolog.Logger } // NewSession creates a new Session for the given connection -func NewSession(server *Server, id int, conn net.Conn) *Session { +func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) return &Session{ @@ -93,6 +95,7 @@ func NewSession(server *Server, id int, conn net.Conn) *Session { reader: reader, remoteHost: host, recipients: make([]*policy.Recipient, 0), + logger: logger, } } @@ -108,17 +111,19 @@ func (s *Session) String() string { * 5. Goto 2 */ func (s *Server) startSession(id int, conn net.Conn) { - log.Infof("SMTP Connection from %v, starting session <%v>", conn.RemoteAddr(), id) + logger := log.With().Str("module", "smtp").Str("remote", conn.RemoteAddr().String()). + Int("session", id).Logger() + logger.Info().Msg("Starting SMTP session") expConnectsCurrent.Add(1) defer func() { if err := conn.Close(); err != nil { - log.Errorf("Error closing connection for <%v>: %v", id, err) + logger.Warn().Err(err).Msg("Closing connection") } s.waitgroup.Done() expConnectsCurrent.Add(-1) }() - ssn := NewSession(s, id, conn) + ssn := NewSession(s, id, conn, logger) ssn.greet() // This is our command reading loop @@ -138,7 +143,7 @@ func (s *Server) startSession(id int, conn net.Conn) { } if !commands[cmd] { ssn.send(fmt.Sprintf("500 Syntax error, %v command unrecognized", cmd)) - ssn.logWarn("Unrecognized command: %v", cmd) + ssn.logger.Warn().Msgf("Unrecognized command: %v", cmd) continue } @@ -147,7 +152,7 @@ func (s *Server) startSession(id int, conn net.Conn) { case "SEND", "SOML", "SAML", "EXPN", "HELP", "TURN": // These commands are not implemented in any state ssn.send(fmt.Sprintf("502 %v command not implemented", cmd)) - ssn.logWarn("Command %v not implemented by Inbucket", cmd) + ssn.logger.Warn().Msgf("Command %v not implemented by Inbucket", cmd) continue case "VRFY": ssn.send("252 Cannot VRFY user, but will accept message") @@ -157,7 +162,7 @@ func (s *Server) startSession(id int, conn net.Conn) { continue case "RSET": // Reset session - ssn.logTrace("Resetting session state on RSET request") + ssn.logger.Debug().Msgf("Resetting session state on RSET request") ssn.reset() ssn.send("250 Session reset") continue @@ -179,7 +184,7 @@ func (s *Server) startSession(id int, conn net.Conn) { ssn.mailHandler(cmd, arg) continue } - ssn.logError("Session entered unexpected state %v", ssn.state) + ssn.logger.Error().Msgf("Session entered unexpected state %v", ssn.state) break } else { ssn.send("500 Syntax error, command garbled") @@ -190,14 +195,14 @@ func (s *Server) startSession(id int, conn net.Conn) { switch ssn.state { case GREET, READY: // EOF is common here - ssn.logInfo("Client closed connection (state %v)", ssn.state) + ssn.logger.Info().Msgf("Client closed connection (state %v)", ssn.state) default: - ssn.logWarn("Got EOF while in state %v", ssn.state) + ssn.logger.Warn().Msgf("Got EOF while in state %v", ssn.state) } break } // not an EOF - ssn.logWarn("Connection error: %v", err) + ssn.logger.Warn().Msgf("Connection error: %v", err) if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { ssn.send("221 Idle timeout, bye bye") @@ -209,9 +214,9 @@ func (s *Server) startSession(id int, conn net.Conn) { } } if ssn.sendError != nil { - ssn.logWarn("Network send error: %v", ssn.sendError) + ssn.logger.Warn().Msgf("Network send error: %v", ssn.sendError) } - ssn.logInfo("Closing connection") + ssn.logger.Info().Msgf("Closing connection") } // GREET state -> waiting for HELO @@ -262,13 +267,13 @@ func (s *Session) readyHandler(cmd string, arg string) { m := re.FindStringSubmatch(arg) if m == nil { s.send("501 Was expecting MAIL arg syntax of FROM:
") - s.logWarn("Bad MAIL argument: %q", arg) + s.logger.Warn().Msgf("Bad MAIL argument: %q", arg) return } from := m[1] if _, _, err := policy.ParseEmailAddress(from); err != nil { s.send("501 Bad sender address syntax") - s.logWarn("Bad address as MAIL arg: %q, %s", from, err) + s.logger.Warn().Msgf("Bad address as MAIL arg: %q, %s", from, err) return } // This is where the client may put BODY=8BITMIME, but we already @@ -277,25 +282,25 @@ func (s *Session) readyHandler(cmd string, arg string) { args, ok := s.parseArgs(m[2]) if !ok { s.send("501 Unable to parse MAIL ESMTP parameters") - s.logWarn("Bad MAIL argument: %q", arg) + s.logger.Warn().Msgf("Bad MAIL argument: %q", arg) return } if args["SIZE"] != "" { size, err := strconv.ParseInt(args["SIZE"], 10, 32) if err != nil { s.send("501 Unable to parse SIZE as an integer") - s.logWarn("Unable to parse SIZE %q as an integer", args["SIZE"]) + s.logger.Warn().Msgf("Unable to parse SIZE %q as an integer", args["SIZE"]) return } if int(size) > s.server.maxMessageBytes { s.send("552 Max message size exceeded") - s.logWarn("Client wanted to send oversized message: %v", args["SIZE"]) + s.logger.Warn().Msgf("Client wanted to send oversized message: %v", args["SIZE"]) return } } } s.from = from - s.logInfo("Mail from: %v", from) + s.logger.Info().Msgf("Mail from: %v", from) s.send(fmt.Sprintf("250 Roger, accepting mail from <%v>", from)) s.enterState(MAIL) } else { @@ -309,7 +314,7 @@ func (s *Session) mailHandler(cmd string, arg string) { case "RCPT": if (len(arg) < 4) || (strings.ToUpper(arg[0:3]) != "TO:") { s.send("501 Was expecting RCPT arg syntax of TO:
") - s.logWarn("Bad RCPT argument: %q", arg) + s.logger.Warn().Msgf("Bad RCPT argument: %q", arg) return } // This trim is probably too forgiving @@ -317,22 +322,22 @@ func (s *Session) mailHandler(cmd string, arg string) { recip, err := s.server.apolicy.NewRecipient(addr) if err != nil { s.send("501 Bad recipient address syntax") - s.logWarn("Bad address as RCPT arg: %q, %s", addr, err) + s.logger.Warn().Msgf("Bad address as RCPT arg: %q, %s", addr, err) return } if len(s.recipients) >= s.server.maxRecips { - s.logWarn("Maximum limit of %v recipients reached", s.server.maxRecips) + s.logger.Warn().Msgf("Maximum limit of %v recipients reached", s.server.maxRecips) s.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", s.server.maxRecips)) return } s.recipients = append(s.recipients, recip) - s.logInfo("Recipient: %v", addr) + s.logger.Info().Msgf("Recipient: %v", addr) s.send(fmt.Sprintf("250 I'll make sure <%v> gets this", addr)) return case "DATA": if arg != "" { s.send("501 DATA command should not have any arguments") - s.logWarn("Got unexpected args on DATA: %q", arg) + s.logger.Warn().Msgf("Got unexpected args on DATA: %q", arg) return } if len(s.recipients) > 0 { @@ -359,7 +364,7 @@ func (s *Session) dataHandler() { s.send("221 Idle timeout, bye bye") } } - s.logWarn("Error: %v while reading", err) + s.logger.Warn().Msgf("Error: %v while reading", err) s.enterState(QUIT) return } @@ -376,7 +381,7 @@ func (s *Session) dataHandler() { _, err := s.server.manager.Deliver( recip, s.from, s.recipients, prefix, msgBuf.Bytes()) if err != nil { - s.logError("delivery for %v: %v", recip.LocalPart, err) + s.logger.Error().Msgf("delivery for %v: %v", recip.LocalPart, err) s.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) s.reset() return @@ -385,7 +390,7 @@ func (s *Session) dataHandler() { expReceivedTotal.Add(1) } s.send("250 Mail accepted for delivery") - s.logInfo("Message size %v bytes", msgBuf.Len()) + s.logger.Info().Msgf("Message size %v bytes", msgBuf.Len()) s.reset() return } @@ -396,7 +401,7 @@ func (s *Session) dataHandler() { msgBuf.Write(lineBuf) if msgBuf.Len() > s.server.maxMessageBytes { s.send("552 Maximum message size exceeded") - s.logWarn("Max message size exceeded while in DATA") + s.logger.Warn().Msgf("Max message size exceeded while in DATA") s.reset() return } @@ -405,7 +410,7 @@ func (s *Session) dataHandler() { func (s *Session) enterState(state State) { s.state = state - s.logTrace("Entering state %v", state) + s.logger.Debug().Msgf("Entering state %v", state) } func (s *Session) greet() { @@ -425,10 +430,10 @@ func (s *Session) send(msg string) { } if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { s.sendError = err - s.logWarn("Failed to send: %q", msg) + s.logger.Warn().Msgf("Failed to send: %q", msg) return } - s.logTrace(">> %v >>", msg) + s.logger.Debug().Msgf(">> %v >>", msg) } // readByteLine reads a line of input, returns byte slice. @@ -448,7 +453,7 @@ func (s *Session) readLine() (line string, err error) { if err != nil { return "", err } - s.logTrace("<< %v <<", strings.TrimRight(line, "\r\n")) + s.logger.Debug().Msgf("<< %v <<", strings.TrimRight(line, "\r\n")) return line, nil } @@ -459,19 +464,19 @@ func (s *Session) parseCmd(line string) (cmd string, arg string, ok bool) { case l == 0: return "", "", true case l < 4: - s.logWarn("Command too short: %q", line) + s.logger.Warn().Msgf("Command too short: %q", line) return "", "", false case l == 4: return strings.ToUpper(line), "", true case l == 5: // Too long to be only command, too short to have args - s.logWarn("Mangled command: %q", line) + s.logger.Warn().Msgf("Mangled command: %q", line) return "", "", false } // If we made it here, command is long enough to have args if line[4] != ' ' { // There wasn't a space after the command? - s.logWarn("Mangled command: %q", line) + s.logger.Warn().Msgf("Mangled command: %q", line) return "", "", false } // I'm not sure if we should trim the args or not, but we will for now @@ -488,13 +493,13 @@ func (s *Session) parseArgs(arg string) (args map[string]string, ok bool) { re := regexp.MustCompile(` (\w+)=(\w+)`) pm := re.FindAllStringSubmatch(arg, -1) if pm == nil { - s.logWarn("Failed to parse arg string: %q") + s.logger.Warn().Msgf("Failed to parse arg string: %q") return nil, false } for _, m := range pm { args[strings.ToUpper(m[1])] = m[2] } - s.logTrace("ESMTP params: %v", args) + s.logger.Debug().Msgf("ESMTP params: %v", args) return args, true } @@ -506,26 +511,5 @@ func (s *Session) reset() { func (s *Session) ooSeq(cmd string) { s.send(fmt.Sprintf("503 Command %v is out of sequence", cmd)) - s.logWarn("Wasn't expecting %v here", cmd) -} - -// Session specific logging methods -func (s *Session) logTrace(msg string, args ...interface{}) { - log.Tracef("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logInfo(msg string, args ...interface{}) { - log.Infof("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logWarn(msg string, args ...interface{}) { - // Update metrics - expWarnsTotal.Add(1) - log.Warnf("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) -} - -func (s *Session) logError(msg string, args ...interface{}) { - // Update metrics - expErrorsTotal.Add(1) - log.Errorf("SMTP[%v]<%v> %v", s.remoteHost, s.id, fmt.Sprintf(msg, args...)) + s.logger.Warn().Msgf("Wasn't expecting %v here", cmd) } From 6601d156be2d066bcf1551d0953cf207f4da6b81 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 12:16:54 -0700 Subject: [PATCH 53/82] metric: new pkg refactored from log for #90 --- pkg/{log/metrics.go => metric/metric.go} | 33 ++++++++++++------------ pkg/server/smtp/listener.go | 14 +++++----- pkg/storage/retention.go | 12 ++++----- 3 files changed, 30 insertions(+), 29 deletions(-) rename pkg/{log/metrics.go => metric/metric.go} (74%) diff --git a/pkg/log/metrics.go b/pkg/metric/metric.go similarity index 74% rename from pkg/log/metrics.go rename to pkg/metric/metric.go index c16f1e8..566445d 100644 --- a/pkg/log/metrics.go +++ b/pkg/metric/metric.go @@ -1,4 +1,4 @@ -package log +package metric import ( "container/list" @@ -7,7 +7,7 @@ import ( "time" ) -// TickerFunc is the type of metrics function accepted by AddTickerFunc +// TickerFunc is the function signature accepted by AddTickerFunc, will be called once per minute. type TickerFunc func() var tickerFuncChan = make(chan TickerFunc) @@ -22,10 +22,10 @@ func AddTickerFunc(f TickerFunc) { tickerFuncChan <- f } -// PushMetric adds the metric to the end of the list and returns a comma separated string of the +// Push adds the metric to the end of the list and returns a comma separated string of the // previous 61 entries. We return 61 instead of 60 (an hour) because the chart on the client // tracks deltas between these values - there is nothing to compare the first value against. -func PushMetric(history *list.List, ev expvar.Var) string { +func Push(history *list.List, ev expvar.Var) string { history.PushBack(ev.String()) if history.Len() > 61 { history.Remove(history.Front()) @@ -33,18 +33,7 @@ func PushMetric(history *list.List, ev expvar.Var) string { return joinStringList(history) } -// joinStringList joins a List containing strings by commas -func joinStringList(listOfStrings *list.List) string { - if listOfStrings.Len() == 0 { - return "" - } - s := make([]string, 0, listOfStrings.Len()) - for e := listOfStrings.Front(); e != nil; e = e.Next() { - s = append(s, e.Value.(string)) - } - return strings.Join(s, ",") -} - +// metricsTicker calls the current list of TickerFuncs once per minute. func metricsTicker() { funcs := make([]TickerFunc, 0) ticker := time.NewTicker(time.Minute) @@ -60,3 +49,15 @@ func metricsTicker() { } } } + +// joinStringList joins a List containing strings by commas. +func joinStringList(listOfStrings *list.List) string { + if listOfStrings.Len() == 0 { + return "" + } + s := make([]string, 0, listOfStrings.Len()) + for e := listOfStrings.Front(); e != nil; e = e.Next() { + s = append(s, e.Value.(string)) + } + return strings.Join(s, ",") +} diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index d566e15..e7fd095 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -11,6 +11,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/metric" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/rs/zerolog/log" ) @@ -27,13 +28,12 @@ func init() { m.Set("WarnsTotal", expWarnsTotal) m.Set("WarnsHist", expWarnsHist) - // TODO #90 move elsewhere - // log.AddTickerFunc(func() { - // expReceivedHist.Set(log.PushMetric(deliveredHist, expReceivedTotal)) - // expConnectsHist.Set(log.PushMetric(connectsHist, expConnectsTotal)) - // expErrorsHist.Set(log.PushMetric(errorsHist, expErrorsTotal)) - // expWarnsHist.Set(log.PushMetric(warnsHist, expWarnsTotal)) - // }) + metric.AddTickerFunc(func() { + expReceivedHist.Set(metric.Push(deliveredHist, expReceivedTotal)) + expConnectsHist.Set(metric.Push(connectsHist, expConnectsTotal)) + expErrorsHist.Set(metric.Push(errorsHist, expErrorsTotal)) + expWarnsHist.Set(metric.Push(warnsHist, expWarnsTotal)) + }) } // Server holds the configuration and state of our SMTP server diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 6a0b714..166da6b 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -7,6 +7,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/metric" "github.com/rs/zerolog/log" ) @@ -42,12 +43,11 @@ func init() { rm.Set("RetainedSize", expRetainedSize) rm.Set("SizeHist", expSizeHist) - // TODO #90 move - // log.AddTickerFunc(func() { - // expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) - // expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) - // expSizeHist.Set(log.PushMetric(sizeHist, expRetainedSize)) - // }) + metric.AddTickerFunc(func() { + expRetentionDeletesHist.Set(metric.Push(retentionDeletesHist, expRetentionDeletesTotal)) + expRetainedHist.Set(metric.Push(retainedHist, expRetainedCurrent)) + expSizeHist.Set(metric.Push(sizeHist, expRetainedSize)) + }) } // RetentionScanner looks for messages older than the configured retention period and deletes them. From cbdb96a4214521e0c595c951f4fa283cb4f986ca Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 12:25:54 -0700 Subject: [PATCH 54/82] log: package deleted for #90 --- pkg/log/logging.go | 162 -------------------------------------- pkg/log/stdout_unix.go | 32 -------- pkg/log/stdout_windows.go | 37 --------- 3 files changed, 231 deletions(-) delete mode 100644 pkg/log/logging.go delete mode 100644 pkg/log/stdout_unix.go delete mode 100644 pkg/log/stdout_windows.go diff --git a/pkg/log/logging.go b/pkg/log/logging.go deleted file mode 100644 index c2184ff..0000000 --- a/pkg/log/logging.go +++ /dev/null @@ -1,162 +0,0 @@ -package log - -import ( - "fmt" - golog "log" - "os" - "strings" - "sync" -) - -// Level is used to indicate the severity of a log entry -type Level int - -const ( - // ERROR indicates a significant problem was encountered - ERROR Level = iota - // WARN indicates something that may be a problem - WARN - // INFO indicates a purely informational log entry - INFO - // TRACE entries are meant for development purposes only - TRACE -) - -var ( - // MaxLevel is the highest Level we will log (max TRACE, min ERROR) - MaxLevel = TRACE - - // logfname is the name of the logfile - logfname string - - // logf is the file we send log output to, will be nil for stderr or stdout - logf *os.File - - mu sync.RWMutex -) - -// Initialize logging. If logfile is equal to "stderr" or "stdout", then -// we will log to that output stream. Otherwise the specificed file will -// opened for writing, and all log data will be placed in it. -func Initialize(logfile string) error { - mu.Lock() - defer mu.Unlock() - if logfile != "stderr" { - // stderr is the go logging default - if logfile == "stdout" { - // set to stdout - golog.SetOutput(os.Stdout) - } else { - logfname = logfile - if err := openLogFile(); err != nil { - return err - } - // Platform specific - closeStdin() - } - } - return nil -} - -// SetLogLevel sets MaxLevel based on the provided string -func SetLogLevel(level string) (ok bool) { - mu.Lock() - defer mu.Unlock() - switch strings.ToUpper(level) { - case "ERROR": - MaxLevel = ERROR - case "WARN": - MaxLevel = WARN - case "INFO": - MaxLevel = INFO - case "TRACE": - MaxLevel = TRACE - default: - golog.Print("Error, unknown log level requested: " + level) - return false - } - return true -} - -// Errorf logs a message to the 'standard' Logger (always), accepts format strings -func Errorf(msg string, args ...interface{}) { - mu.RLock() - defer mu.RUnlock() - msg = "[ERROR] " + msg - golog.Printf(msg, args...) -} - -// Warnf logs a message to the 'standard' Logger if MaxLevel is >= WARN, accepts format strings -func Warnf(msg string, args ...interface{}) { - mu.RLock() - defer mu.RUnlock() - if MaxLevel >= WARN { - msg = "[WARN ] " + msg - golog.Printf(msg, args...) - } -} - -// Infof logs a message to the 'standard' Logger if MaxLevel is >= INFO, accepts format strings -func Infof(msg string, args ...interface{}) { - mu.RLock() - defer mu.RUnlock() - if MaxLevel >= INFO { - msg = "[INFO ] " + msg - golog.Printf(msg, args...) - } -} - -// Tracef logs a message to the 'standard' Logger if MaxLevel is >= TRACE, accepts format strings -func Tracef(msg string, args ...interface{}) { - mu.RLock() - defer mu.RUnlock() - if MaxLevel >= TRACE { - msg = "[TRACE] " + msg - golog.Printf(msg, args...) - } -} - -// Rotate closes the current log file, then reopens it. This gives an external -// log rotation system the opportunity to move the existing log file out of the -// way and have Inbucket create a new one. -func Rotate() { - mu.Lock() - defer mu.Unlock() - // Rotate logs if configured - if logf != nil { - closeLogFile() - // There is nothing we can do if the log open fails - _ = openLogFile() - } else { - Infof("Ignoring SIGHUP, logfile not configured") - } -} - -// Close the log file if we have one open -func Close() { - mu.Lock() - defer mu.Unlock() - if logf != nil { - closeLogFile() - } -} - -// openLogFile creates or appends to the logfile passed on commandline -func openLogFile() error { - // use specified log file - var err error - logf, err = os.OpenFile(logfname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) - if err != nil { - return fmt.Errorf("failed to create %v: %v", logfname, err) - } - golog.SetOutput(logf) - // Platform specific - reassignStdout() - return nil -} - -// closeLogFile closes the current logfile -func closeLogFile() { - // We are never in a situation where we can do anything about failing to close - _ = logf.Close() -} diff --git a/pkg/log/stdout_unix.go b/pkg/log/stdout_unix.go deleted file mode 100644 index 0908102..0000000 --- a/pkg/log/stdout_unix.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build !windows - -package log - -import ( - "log" - "os" - - "golang.org/x/sys/unix" -) - -// closeStdin will close stdin on Unix platforms - this is standard practice -// for daemons -func closeStdin() { - if err := os.Stdin.Close(); err != nil { - // Not a fatal error - log.Printf("Failed to close os.Stdin during log setup") - } -} - -// reassignStdout points stdout/stderr to our logfile on systems that support -// the Dup2 syscall per https://github.com/golang/go/issues/325 -func reassignStdout() { - if err := unix.Dup2(int(logf.Fd()), 1); err != nil { - // Not considered fatal - log.Printf("Failed to re-assign stdout to logfile: %v", err) - } - if err := unix.Dup2(int(logf.Fd()), 2); err != nil { - // Not considered fatal - log.Printf("Failed to re-assign stderr to logfile: %v", err) - } -} diff --git a/pkg/log/stdout_windows.go b/pkg/log/stdout_windows.go deleted file mode 100644 index f883d24..0000000 --- a/pkg/log/stdout_windows.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build windows - -package log - -import ( - "log" - "os" -) - -var stdOutsClosed = false - -// closeStdin does nothing on Windows, it would always fail -func closeStdin() { - // Nop -} - -// reassignStdout points stdout/stderr to our logfile on systems that do not -// support the Dup2 syscall -func reassignStdout() { - if !stdOutsClosed { - // Close std* streams to prevent accidental output, they will be redirected to - // our logfile below - - // Warning: this will hide panic() output, sorry Windows users - if err := os.Stderr.Close(); err != nil { - // Not considered fatal - log.Printf("Failed to close os.Stderr during log setup") - } - if err := os.Stdin.Close(); err != nil { - // Not considered fatal - log.Printf("Failed to close os.Stdin during log setup") - } - os.Stdout = logf - os.Stderr = logf - stdOutsClosed = true - } -} From 92f2da5025bba69782be4ca31d20a32cad01aedf Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 13:37:42 -0700 Subject: [PATCH 55/82] server: -netdebug flag now controls tracing for #90 Network trace is sent to stdout, no longer part of normal debug logging. --- cmd/inbucket/main.go | 5 +++ pkg/config/config.go | 2 + pkg/server/pop3/handler.go | 77 ++++++++++++++----------------------- pkg/server/pop3/listener.go | 3 ++ pkg/server/smtp/handler.go | 18 +++++++-- pkg/server/smtp/listener.go | 3 ++ 6 files changed, 56 insertions(+), 52 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 9423128..4f47434 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -59,6 +59,7 @@ func main() { pidfile := flag.String("pidfile", "", "Write our PID into the specified file.") logfile := flag.String("logfile", "stderr", "Write out log into the specified file.") logjson := flag.Bool("logjson", false, "Logs are written in JSON format.") + netdebug := flag.Bool("netdebug", false, "Dump SMTP & POP3 network traffic to stdout.") flag.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: inbucket [options]") flag.PrintDefaults() @@ -89,6 +90,10 @@ func main() { fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err) os.Exit(1) } + if *netdebug { + conf.POP3.Debug = true + conf.SMTP.Debug = true + } // Setup signal handler. sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) diff --git a/pkg/config/config.go b/pkg/config/config.go index dd35f39..7db13a8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -45,6 +45,7 @@ type SMTP struct { MaxMessageBytes int `required:"true" default:"10240000" desc:"Maximum message size"` StoreMessages bool `required:"true" default:"true" desc:"Store incoming mail?"` Timeout time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` + Debug bool `ignored:"true"` } // POP3 contains the POP3 server configuration. @@ -52,6 +53,7 @@ type POP3 struct { Addr string `required:"true" default:"0.0.0.0:1100" desc:"POP3 server IP4 host:port"` Domain string `required:"true" default:"inbucket" desc:"HELLO domain"` Timeout time.Duration `required:"true" default:"600s" desc:"Idle network timeout"` + Debug bool `ignored:"true"` } // Web contains the HTTP server configuration. diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 8e8eeb8..cbf3628 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -2,7 +2,6 @@ package pop3 import ( "bufio" - "bytes" "fmt" "io" "net" @@ -58,26 +57,35 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - messages []storage.Message // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages - logger zerolog.Logger + server *Server // Reference to the server we belong to. + id int // Session ID number. + conn net.Conn // Our network connection. + remoteHost string // IP address of client. + sendError error // Used to bail out of read loop on send error. + state State // Current session state. + reader *bufio.Reader // Buffered reader for our net conn. + user string // Mailbox name. + messages []storage.Message // Slice of messages in mailbox. + retain []bool // Messages to retain upon UPDATE (true=retain). + msgCount int // Number of undeleted messages. + logger zerolog.Logger // Session specific logger. + debug bool // Print network traffic to stdout. } // NewSession creates a new POP3 session func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) - return &Session{server: server, id: id, conn: conn, state: AUTHORIZATION, - reader: reader, remoteHost: host, logger: logger} + return &Session{ + server: server, + id: id, + conn: conn, + state: AUTHORIZATION, + reader: reader, + remoteHost: host, + logger: logger, + debug: server.config.Debug, + } } func (s *Session) String() string { @@ -550,41 +558,12 @@ func (s *Session) send(msg string) { } if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { s.sendError = err - s.logger.Warn().Msgf("Failed to send: '%v'", msg) + s.logger.Warn().Msgf("Failed to send: %q", msg) return } - s.logger.Debug().Msgf(">> %v >>", msg) -} - -// readByteLine reads a line of input into the provided buffer. Does -// not reset the Buffer - please do so prior to calling. -func (s *Session) readByteLine(buf *bytes.Buffer) error { - if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { - return err + if s.debug { + fmt.Printf("%04d > %v\n", s.id, msg) } - for { - line, err := s.reader.ReadBytes('\r') - if err != nil { - return err - } - if _, err = buf.Write(line); err != nil { - return err - } - // Read the next byte looking for '\n' - c, err := s.reader.ReadByte() - if err != nil { - return err - } - if err := buf.WriteByte(c); err != nil { - return err - } - if c == '\n' { - // We've reached the end of the line, return - return nil - } - // Else, keep looking - } - // Should be unreachable } // Reads a line of input @@ -596,7 +575,9 @@ func (s *Session) readLine() (line string, err error) { if err != nil { return "", err } - s.logger.Debug().Msgf("<< %v <<", strings.TrimRight(line, "\r\n")) + if s.debug { + fmt.Printf("%04d %v\n", s.id, strings.TrimRight(line, "\r\n")) + } return line, nil } diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index bf9d6fd..cf4c14a 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -13,6 +13,8 @@ import ( // Server defines an instance of our POP3 server type Server struct { + // TODO(#91) Refactor config items out of this struct + config config.POP3 host string domain string timeout time.Duration @@ -25,6 +27,7 @@ type Server struct { // New creates a new Server struct func New(cfg config.POP3, shutdownChan chan bool, store storage.Store) *Server { return &Server{ + config: cfg, host: cfg.Addr, domain: cfg.Domain, store: store, diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index d696776..fea9817 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -80,7 +80,8 @@ type Session struct { reader *bufio.Reader from string recipients []*policy.Recipient - logger zerolog.Logger + logger zerolog.Logger // Session specific logger. + debug bool // Print network traffic to stdout. } // NewSession creates a new Session for the given connection @@ -96,6 +97,7 @@ func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *S remoteHost: host, recipients: make([]*policy.Recipient, 0), logger: logger, + debug: server.config.Debug, } } @@ -433,7 +435,9 @@ func (s *Session) send(msg string) { s.logger.Warn().Msgf("Failed to send: %q", msg) return } - s.logger.Debug().Msgf(">> %v >>", msg) + if s.debug { + fmt.Printf("%04d > %v\n", s.id, msg) + } } // readByteLine reads a line of input, returns byte slice. @@ -441,7 +445,11 @@ func (s *Session) readByteLine() ([]byte, error) { if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return nil, err } - return s.reader.ReadBytes('\n') + b, err := s.reader.ReadBytes('\n') + if err == nil && s.debug { + fmt.Printf("%04d %s\n", s.id, bytes.TrimRight(b, "\r\n")) + } + return b, err } // Reads a line of input @@ -453,7 +461,9 @@ func (s *Session) readLine() (line string, err error) { if err != nil { return "", err } - s.logger.Debug().Msgf("<< %v <<", strings.TrimRight(line, "\r\n")) + if s.debug { + fmt.Printf("%04d %v\n", s.id, strings.TrimRight(line, "\r\n")) + } return line, nil } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index e7fd095..e047520 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -38,6 +38,8 @@ func init() { // Server holds the configuration and state of our SMTP server type Server struct { + // TODO(#91) Refactor config items out of this struct + config config.SMTP // Configuration host string domain string @@ -86,6 +88,7 @@ func NewServer( apolicy *policy.Addressing, ) *Server { return &Server{ + config: cfg, host: cfg.Addr, domain: cfg.Domain, domainNoStore: strings.ToLower(cfg.DomainNoStore), From e076f8041662e59b8c3cad3741060277dc22d552 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 14:06:58 -0700 Subject: [PATCH 56/82] smtp: Use zerolog hooks for warns/errors expvars #90 --- pkg/server/smtp/handler.go | 4 +++- pkg/server/smtp/listener.go | 42 ++++++++++++++++++------------------- pkg/server/smtp/loghook.go | 15 +++++++++++++ 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 pkg/server/smtp/loghook.go diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index fea9817..a86be5e 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -113,7 +113,9 @@ func (s *Session) String() string { * 5. Goto 2 */ func (s *Server) startSession(id int, conn net.Conn) { - logger := log.With().Str("module", "smtp").Str("remote", conn.RemoteAddr().String()). + logger := log.Hook(logHook{}).With(). + Str("module", "smtp"). + Str("remote", conn.RemoteAddr().String()). Int("session", id).Logger() logger.Info().Msg("Starting SMTP session") expConnectsCurrent.Add(1) diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index e047520..e3a63e2 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -36,6 +36,27 @@ func init() { }) } +var ( + // Raw stat collectors + expConnectsTotal = new(expvar.Int) + expConnectsCurrent = new(expvar.Int) + expReceivedTotal = new(expvar.Int) + expErrorsTotal = new(expvar.Int) + expWarnsTotal = new(expvar.Int) + + // History of certain stats + deliveredHist = list.New() + connectsHist = list.New() + errorsHist = list.New() + warnsHist = list.New() + + // History rendered as comma delim string + expReceivedHist = new(expvar.String) + expConnectsHist = new(expvar.String) + expErrorsHist = new(expvar.String) + expWarnsHist = new(expvar.String) +) + // Server holds the configuration and state of our SMTP server type Server struct { // TODO(#91) Refactor config items out of this struct @@ -59,27 +80,6 @@ type Server struct { waitgroup *sync.WaitGroup // Waitgroup tracks individual sessions } -var ( - // Raw stat collectors - expConnectsTotal = new(expvar.Int) - expConnectsCurrent = new(expvar.Int) - expReceivedTotal = new(expvar.Int) - expErrorsTotal = new(expvar.Int) - expWarnsTotal = new(expvar.Int) - - // History of certain stats - deliveredHist = list.New() - connectsHist = list.New() - errorsHist = list.New() - warnsHist = list.New() - - // History rendered as comma delim string - expReceivedHist = new(expvar.String) - expConnectsHist = new(expvar.String) - expErrorsHist = new(expvar.String) - expWarnsHist = new(expvar.String) -) - // NewServer creates a new Server instance with the specificed config func NewServer( cfg config.SMTP, diff --git a/pkg/server/smtp/loghook.go b/pkg/server/smtp/loghook.go new file mode 100644 index 0000000..59f69da --- /dev/null +++ b/pkg/server/smtp/loghook.go @@ -0,0 +1,15 @@ +package smtp + +import "github.com/rs/zerolog" + +type logHook struct{} + +// Run implements a zerolog hook that updates the SMTP warning/error expvars. +func (h logHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { + switch level { + case zerolog.WarnLevel: + expWarnsTotal.Add(1) + case zerolog.ErrorLevel: + expErrorsTotal.Add(1) + } +} From deceb29377ca3c12ea306016ccf003f5ed9bf366 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 15:16:22 -0700 Subject: [PATCH 57/82] inbucket: respect -logfile flag again for #90 Removed log file rotation, too racy, not needed in the world of docker and systemd. --- CHANGELOG.md | 7 +++++ cmd/inbucket/main.go | 67 +++++++++++++++++++++++++++++++++----------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 888adb8..f1872bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,13 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Uses the same default ports as other builds; smtp:2500 http:9000 pop3:1100 - Uses volume `/config` for `greeting.html` - Uses volume `/storage` for mail storage +- Log output is now structured, and will be output as JSON with the `-logjson` + flag; which is enabled by default for the Docker container. +- SMTP and POP3 network tracing is no longer logged regardless of level, but can + be sent to stdout via `-netdebug` flag. + +### Removed +- Support for SIGHUP and log file rotation. ## [v1.3.1] - 2018-03-10 diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 4f47434..5622d8f 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -2,10 +2,12 @@ package main import ( + "bufio" "context" "expvar" "flag" "fmt" + "io" "os" "os/signal" "runtime" @@ -72,16 +74,12 @@ func main() { return } // Logger setup. - if !*logjson { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) - zerolog.SetGlobalLevel(zerolog.DebugLevel) + closeLog, err := openLog(*logfile, *logjson) + if err != nil { + fmt.Fprintf(os.Stderr, "Log error: %v\n", err) + os.Exit(1) } - _ = logfile - // } else if *logfile != "stderr" { - // // TODO #90 file output - // // defer close - // } - slog := log.With().Str("phase", "startup").Logger() + startupLog := log.With().Str("phase", "startup").Logger() // Process configuration. config.Version = version config.BuildDate = date @@ -96,18 +94,19 @@ func main() { } // Setup signal handler. sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) // Initialize logging. - slog.Info().Str("version", config.Version).Str("buildDate", config.BuildDate).Msg("Inbucket") + startupLog.Info().Str("version", config.Version).Str("buildDate", config.BuildDate). + Msg("Inbucket starting") // Write pidfile if requested. if *pidfile != "" { pidf, err := os.Create(*pidfile) if err != nil { - slog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to create pidfile") + startupLog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to create pidfile") } fmt.Fprintf(pidf, "%v\n", os.Getpid()) if err := pidf.Close(); err != nil { - slog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to close pidfile") + startupLog.Fatal().Err(err).Str("path", *pidfile).Msg("Failed to close pidfile") } } // Configure internal services. @@ -116,7 +115,7 @@ func main() { store, err := storage.FromConfig(conf.Storage) if err != nil { removePIDFile(*pidfile) - slog.Fatal().Err(err).Str("module", "storage").Msg("Fatal storage error") + startupLog.Fatal().Err(err).Str("module", "storage").Msg("Fatal storage error") } msgHub := msghub.New(rootCtx, conf.Web.MonitorHistory) addrPolicy := &policy.Addressing{Config: conf.SMTP} @@ -141,9 +140,6 @@ signalLoop: select { case sig := <-sigChan: switch sig { - case syscall.SIGHUP: - log.Info().Str("signal", "SIGHUP").Msg("Recieved SIGHUP, cycling logfile") - // TODO #90 log.Rotate() case syscall.SIGINT: // Shutdown requested log.Info().Str("phase", "shutdown").Str("signal", "SIGINT"). @@ -166,6 +162,43 @@ signalLoop: pop3Server.Drain() retentionScanner.Join() removePIDFile(*pidfile) + closeLog() +} + +// openLog configures zerolog output, returns func to close logfile. +func openLog(logfile string, json bool) (close func(), err error) { + close = func() {} + zerolog.SetGlobalLevel(zerolog.DebugLevel) + var w io.Writer + color := true + switch logfile { + case "stderr": + w = os.Stderr + case "stdout": + w = os.Stdout + default: + logf, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + return nil, err + } + bw := bufio.NewWriter(logf) + w = bw + color = false + close = func() { + _ = bw.Flush() + _ = logf.Close() + } + } + w = zerolog.SyncWriter(w) + if json { + log.Logger = log.Output(w) + return close, nil + } + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: w, + NoColor: !color, + }) + return close, nil } // removePIDFile removes the PID file if created. From 5a28e9f9e733ffaf401424a2aa725cd230f007c3 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 15:30:36 -0700 Subject: [PATCH 58/82] config: Use log level name DEBUG instead of TRACE Add log level parsing into openLog() for #90 --- cmd/inbucket/main.go | 30 +++++++++++++++++++++--------- doc/config.md | 4 ++-- pkg/config/config.go | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 5622d8f..87c03cd 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "runtime" + "strings" "syscall" "time" @@ -73,13 +74,6 @@ func main() { config.Usage() return } - // Logger setup. - closeLog, err := openLog(*logfile, *logjson) - if err != nil { - fmt.Fprintf(os.Stderr, "Log error: %v\n", err) - os.Exit(1) - } - startupLog := log.With().Str("phase", "startup").Logger() // Process configuration. config.Version = version config.BuildDate = date @@ -92,6 +86,13 @@ func main() { conf.POP3.Debug = true conf.SMTP.Debug = true } + // Logger setup. + closeLog, err := openLog(conf.LogLevel, *logfile, *logjson) + if err != nil { + fmt.Fprintf(os.Stderr, "Log error: %v\n", err) + os.Exit(1) + } + startupLog := log.With().Str("phase", "startup").Logger() // Setup signal handler. sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) @@ -166,9 +167,20 @@ signalLoop: } // openLog configures zerolog output, returns func to close logfile. -func openLog(logfile string, json bool) (close func(), err error) { +func openLog(level string, logfile string, json bool) (close func(), err error) { + switch strings.ToUpper(level) { + case "DEBUG": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "INFO": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "WARN": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "ERROR": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + default: + return nil, fmt.Errorf("Log level %q not one of: DEBUG, INFO, WARN, ERROR", level) + } close = func() {} - zerolog.SetGlobalLevel(zerolog.DebugLevel) var w io.Writer color := true switch logfile { diff --git a/doc/config.md b/doc/config.md index 5f9fa92..8f8c5ba 100644 --- a/doc/config.md +++ b/doc/config.md @@ -8,7 +8,7 @@ Running `inbucket -help` will yield a condensed summary of the environment variables it supports: KEY DEFAULT DESCRIPTION - INBUCKET_LOGLEVEL INFO TRACE, INFO, WARN, or ERROR + INBUCKET_LOGLEVEL INFO DEBUG, INFO, WARN, or ERROR INBUCKET_SMTP_ADDR 0.0.0.0:2500 SMTP server IP4 host:port INBUCKET_SMTP_DOMAIN inbucket HELO domain INBUCKET_SMTP_DOMAINNOSTORE Load testing domain @@ -47,7 +47,7 @@ should probably select INFO, but a busy shared installation would be better off with WARN or ERROR. - Default: `INFO` -- Values: one of `TRACE`, `INFO`, `WARN`, or `ERROR` +- Values: one of `DEBUG`, `INFO`, `WARN`, or `ERROR` ## SMTP diff --git a/pkg/config/config.go b/pkg/config/config.go index 7db13a8..a8452f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -29,7 +29,7 @@ var ( // Root wraps all other configurations. type Root struct { - LogLevel string `required:"true" default:"INFO" desc:"TRACE, INFO, WARN, or ERROR"` + LogLevel string `required:"true" default:"INFO" desc:"DEBUG, INFO, WARN, or ERROR"` SMTP SMTP POP3 POP3 Web Web From 47b526824b8dceb6e9caab4575d1bd7a65a980f4 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 15:36:39 -0700 Subject: [PATCH 59/82] travis: Move to Go 1.10.x --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 817ab3f..f45a552 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ before_script: - make deps go: - - "1.10" + - "1.10.x" deploy: provider: script From 87bab63aa2451f3274407fc185a0f0ed415fdc4a Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 16:11:12 -0700 Subject: [PATCH 60/82] docker: Default to JSON log output for #90 --- Dockerfile | 3 ++- Makefile | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5bc5da5..b716802 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,8 @@ VOLUME /config VOLUME /storage WORKDIR $INBUCKET_HOME -ENTRYPOINT "/start-inbucket.sh" +ENTRYPOINT ["/start-inbucket.sh"] +CMD ["-logjson"] # Build Inbucket COPY . $INBUCKET_SRC/ diff --git a/Makefile b/Makefile index ff275c0..857fca5 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ $(commands): %: cmd/% clean: go clean $(PKGS) rm -f $(commands) + rm -rf dist deps: go get -t ./... From 2c813081ebd20a4106db7e41fdd517fb3747ed9d Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 16:49:52 -0700 Subject: [PATCH 61/82] smtp: Use config.SMTP directly in Server #91 --- pkg/config/config.go | 2 + pkg/server/smtp/handler.go | 26 ++++---- pkg/server/smtp/handler_test.go | 2 +- pkg/server/smtp/listener.go | 102 +++++++++++++------------------- 4 files changed, 57 insertions(+), 75 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index a8452f1..3dc7549 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,6 +3,7 @@ package config import ( "log" "os" + "strings" "text/tabwriter" "time" @@ -81,6 +82,7 @@ type Storage struct { func Process() (*Root, error) { c := &Root{} err := envconfig.Process(prefix, c) + c.SMTP.DomainNoStore = strings.ToLower(c.SMTP.DomainNoStore) return c, err } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index a86be5e..9d4f238 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -123,7 +123,7 @@ func (s *Server) startSession(id int, conn net.Conn) { if err := conn.Close(); err != nil { logger.Warn().Err(err).Msg("Closing connection") } - s.waitgroup.Done() + s.wg.Done() expConnectsCurrent.Add(-1) }() @@ -244,7 +244,7 @@ func (s *Session) greetHandler(cmd string, arg string) { s.remoteDomain = domain s.send("250-Great, let's get this show on the road") s.send("250-8BITMIME") - s.send(fmt.Sprintf("250 SIZE %v", s.server.maxMessageBytes)) + s.send(fmt.Sprintf("250 SIZE %v", s.server.config.MaxMessageBytes)) s.enterState(READY) default: s.ooSeq(cmd) @@ -296,7 +296,7 @@ func (s *Session) readyHandler(cmd string, arg string) { s.logger.Warn().Msgf("Unable to parse SIZE %q as an integer", args["SIZE"]) return } - if int(size) > s.server.maxMessageBytes { + if int(size) > s.server.config.MaxMessageBytes { s.send("552 Max message size exceeded") s.logger.Warn().Msgf("Client wanted to send oversized message: %v", args["SIZE"]) return @@ -323,15 +323,17 @@ func (s *Session) mailHandler(cmd string, arg string) { } // This trim is probably too forgiving addr := strings.Trim(arg[3:], "<> ") - recip, err := s.server.apolicy.NewRecipient(addr) + recip, err := s.server.addrPolicy.NewRecipient(addr) if err != nil { s.send("501 Bad recipient address syntax") s.logger.Warn().Msgf("Bad address as RCPT arg: %q, %s", addr, err) return } - if len(s.recipients) >= s.server.maxRecips { - s.logger.Warn().Msgf("Maximum limit of %v recipients reached", s.server.maxRecips) - s.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", s.server.maxRecips)) + if len(s.recipients) >= s.server.config.MaxRecipients { + s.logger.Warn().Msgf("Maximum limit of %v recipients reached", + s.server.config.MaxRecipients) + s.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", + s.server.config.MaxRecipients)) return } s.recipients = append(s.recipients, recip) @@ -379,7 +381,7 @@ func (s *Session) dataHandler() { if recip.ShouldStore() { // Generate Received header. prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - s.remoteDomain, s.remoteHost, s.server.domain, recip.Address.Address, + s.remoteDomain, s.remoteHost, s.server.config.Domain, recip.Address.Address, tstamp) // Deliver message. _, err := s.server.manager.Deliver( @@ -403,7 +405,7 @@ func (s *Session) dataHandler() { lineBuf = lineBuf[1:] } msgBuf.Write(lineBuf) - if msgBuf.Len() > s.server.maxMessageBytes { + if msgBuf.Len() > s.server.config.MaxMessageBytes { s.send("552 Maximum message size exceeded") s.logger.Warn().Msgf("Max message size exceeded while in DATA") s.reset() @@ -418,12 +420,12 @@ func (s *Session) enterState(state State) { } func (s *Session) greet() { - s.send(fmt.Sprintf("220 %v Inbucket SMTP ready", s.server.domain)) + s.send(fmt.Sprintf("220 %v Inbucket SMTP ready", s.server.config.Domain)) } -// Calculate the next read or write deadline based on maxIdle +// nextDeadline calculates the next read or write deadline based on configured timeout. func (s *Session) nextDeadline() time.Time { - return time.Now().Add(s.server.timeout) + return time.Now().Add(s.server.config.Timeout) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 27fa663..e1b75d8 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -392,7 +392,7 @@ func setupSMTPSession(server *Server) net.Conn { // Pair of pipes to communicate serverConn, clientConn := net.Pipe() // Start the session - server.waitgroup.Add(1) + server.wg.Add(1) sessionNum++ go server.startSession(sessionNum, &mockConn{serverConn}) diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index e3a63e2..3741eef 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -5,7 +5,6 @@ import ( "context" "expvar" "net" - "strings" "sync" "time" @@ -16,26 +15,6 @@ import ( "github.com/rs/zerolog/log" ) -func init() { - m := expvar.NewMap("smtp") - m.Set("ConnectsTotal", expConnectsTotal) - m.Set("ConnectsHist", expConnectsHist) - m.Set("ConnectsCurrent", expConnectsCurrent) - m.Set("ReceivedTotal", expReceivedTotal) - m.Set("ReceivedHist", expReceivedHist) - m.Set("ErrorsTotal", expErrorsTotal) - m.Set("ErrorsHist", expErrorsHist) - m.Set("WarnsTotal", expWarnsTotal) - m.Set("WarnsHist", expWarnsHist) - - metric.AddTickerFunc(func() { - expReceivedHist.Set(metric.Push(deliveredHist, expReceivedTotal)) - expConnectsHist.Set(metric.Push(connectsHist, expConnectsTotal)) - expErrorsHist.Set(metric.Push(errorsHist, expErrorsTotal)) - expWarnsHist.Set(metric.Push(warnsHist, expWarnsTotal)) - }) -} - var ( // Raw stat collectors expConnectsTotal = new(expvar.Int) @@ -57,56 +36,55 @@ var ( expWarnsHist = new(expvar.String) ) -// Server holds the configuration and state of our SMTP server -type Server struct { - // TODO(#91) Refactor config items out of this struct - config config.SMTP - // Configuration - host string - domain string - domainNoStore string - maxRecips int - maxMessageBytes int - storeMessages bool - timeout time.Duration - - // Dependencies - apolicy *policy.Addressing // Address policy. - globalShutdown chan bool // Shuts down Inbucket. - manager message.Manager // Used to deliver messages. - - // State - listener net.Listener // Incoming network connections - waitgroup *sync.WaitGroup // Waitgroup tracks individual sessions +func init() { + m := expvar.NewMap("smtp") + m.Set("ConnectsTotal", expConnectsTotal) + m.Set("ConnectsHist", expConnectsHist) + m.Set("ConnectsCurrent", expConnectsCurrent) + m.Set("ReceivedTotal", expReceivedTotal) + m.Set("ReceivedHist", expReceivedHist) + m.Set("ErrorsTotal", expErrorsTotal) + m.Set("ErrorsHist", expErrorsHist) + m.Set("WarnsTotal", expWarnsTotal) + m.Set("WarnsHist", expWarnsHist) + metric.AddTickerFunc(func() { + expReceivedHist.Set(metric.Push(deliveredHist, expReceivedTotal)) + expConnectsHist.Set(metric.Push(connectsHist, expConnectsTotal)) + expErrorsHist.Set(metric.Push(errorsHist, expErrorsTotal)) + expWarnsHist.Set(metric.Push(warnsHist, expWarnsTotal)) + }) } -// NewServer creates a new Server instance with the specificed config +// Server holds the configuration and state of our SMTP server. +type Server struct { + config config.SMTP // SMTP configuration. + addrPolicy *policy.Addressing // Address policy. + globalShutdown chan bool // Shuts down Inbucket. + manager message.Manager // Used to deliver messages. + listener net.Listener // Incoming network connections. + wg *sync.WaitGroup // Waitgroup tracks individual sessions. +} + +// NewServer creates a new Server instance with the specificed config. func NewServer( - cfg config.SMTP, + smtpConfig config.SMTP, globalShutdown chan bool, manager message.Manager, apolicy *policy.Addressing, ) *Server { return &Server{ - config: cfg, - host: cfg.Addr, - domain: cfg.Domain, - domainNoStore: strings.ToLower(cfg.DomainNoStore), - maxRecips: cfg.MaxRecipients, - timeout: cfg.Timeout, - maxMessageBytes: cfg.MaxMessageBytes, - storeMessages: cfg.StoreMessages, - globalShutdown: globalShutdown, - manager: manager, - apolicy: apolicy, - waitgroup: new(sync.WaitGroup), + config: smtpConfig, + globalShutdown: globalShutdown, + manager: manager, + addrPolicy: apolicy, + wg: new(sync.WaitGroup), } } // Start the listener and handle incoming connections. func (s *Server) Start(ctx context.Context) { slog := log.With().Str("module", "smtp").Str("phase", "startup").Logger() - addr, err := net.ResolveTCPAddr("tcp4", s.host) + addr, err := net.ResolveTCPAddr("tcp4", s.config.Addr) if err != nil { slog.Error().Err(err).Msg("Failed to build tcp4 address") s.emergencyShutdown() @@ -119,10 +97,10 @@ func (s *Server) Start(ctx context.Context) { s.emergencyShutdown() return } - if !s.storeMessages { + if !s.config.StoreMessages { slog.Info().Msg("Load test mode active, messages will not be stored") - } else if s.domainNoStore != "" { - slog.Info().Msgf("Messages sent to domain '%v' will be discarded", s.domainNoStore) + } else if s.config.DomainNoStore != "" { + slog.Info().Msgf("Messages sent to domain '%v' will be discarded", s.config.DomainNoStore) } // Listener go routine. go s.serve(ctx) @@ -172,7 +150,7 @@ func (s *Server) serve(ctx context.Context) { } else { tempDelay = 0 expConnectsTotal.Add(1) - s.waitgroup.Add(1) + s.wg.Add(1) go s.startSession(sessionID, conn) } } @@ -190,6 +168,6 @@ func (s *Server) emergencyShutdown() { // Drain causes the caller to block until all active SMTP sessions have finished func (s *Server) Drain() { // Wait for sessions to close. - s.waitgroup.Wait() + s.wg.Wait() log.Debug().Str("module", "smtp").Str("phase", "shutdown").Msg("SMTP connections have drained") } From 7b073562eb5ea153797135816360c6f59b90ebb1 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 17:01:02 -0700 Subject: [PATCH 62/82] pop3: Use config.POP3 directly in server #91 --- pkg/server/pop3/handler.go | 8 ++++---- pkg/server/pop3/listener.go | 33 +++++++++++++-------------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index cbf3628..bd5c07c 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -107,12 +107,12 @@ func (s *Server) startSession(id int, conn net.Conn) { if err := conn.Close(); err != nil { logger.Warn().Err(err).Msg("Closing connection") } - s.waitgroup.Done() + s.wg.Done() }() ssn := NewSession(s, id, conn, logger) ssn.send(fmt.Sprintf("+OK Inbucket POP3 server ready <%v.%v@%v>", os.Getpid(), - time.Now().Unix(), s.domain)) + time.Now().Unix(), s.config.Domain)) // This is our command reading loop for ssn.state != QUIT && ssn.sendError == nil { @@ -545,9 +545,9 @@ func (s *Session) enterState(state State) { s.logger.Debug().Msgf("Entering state %v", state) } -// Calculate the next read or write deadline based on maxIdleSeconds +// nextDeadline calculates the next read or write deadline based on configured timeout. func (s *Session) nextDeadline() time.Time { - return time.Now().Add(s.server.timeout) + return time.Now().Add(s.server.config.Timeout) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index cf4c14a..c9a652f 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -11,36 +11,29 @@ import ( "github.com/rs/zerolog/log" ) -// Server defines an instance of our POP3 server +// Server defines an instance of the POP3 server. type Server struct { - // TODO(#91) Refactor config items out of this struct - config config.POP3 - host string - domain string - timeout time.Duration - store storage.Store - listener net.Listener - globalShutdown chan bool - waitgroup *sync.WaitGroup + config config.POP3 // POP3 configuration. + store storage.Store // Mail store. + listener net.Listener // TCP listener. + globalShutdown chan bool // Inbucket shutdown signal. + wg *sync.WaitGroup // Waitgroup tracking sessions. } -// New creates a new Server struct -func New(cfg config.POP3, shutdownChan chan bool, store storage.Store) *Server { +// New creates a new Server struct. +func New(pop3Config config.POP3, shutdownChan chan bool, store storage.Store) *Server { return &Server{ - config: cfg, - host: cfg.Addr, - domain: cfg.Domain, + config: pop3Config, store: store, - timeout: cfg.Timeout, globalShutdown: shutdownChan, - waitgroup: new(sync.WaitGroup), + wg: new(sync.WaitGroup), } } // Start the server and listen for connections func (s *Server) Start(ctx context.Context) { slog := log.With().Str("module", "pop3").Str("phase", "startup").Logger() - addr, err := net.ResolveTCPAddr("tcp4", s.host) + addr, err := net.ResolveTCPAddr("tcp4", s.config.Addr) if err != nil { slog.Error().Err(err).Msg("Failed to build tcp4 address") s.emergencyShutdown() @@ -101,7 +94,7 @@ func (s *Server) serve(ctx context.Context) { } } else { tempDelay = 0 - s.waitgroup.Add(1) + s.wg.Add(1) go s.startSession(sid, conn) } } @@ -119,6 +112,6 @@ func (s *Server) emergencyShutdown() { // Drain causes the caller to block until all active POP3 sessions have finished func (s *Server) Drain() { // Wait for sessions to close - s.waitgroup.Wait() + s.wg.Wait() log.Debug().Str("module", "pop3").Str("phase", "shutdown").Msg("POP3 connections have drained") } From 3fe41407334661abe9c306c7bdad45abf0047ebf Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 17:09:30 -0700 Subject: [PATCH 63/82] pop3, smtp: embed Server struct into Session for #91 --- pkg/server/pop3/handler.go | 10 ++++---- pkg/server/smtp/handler.go | 48 +++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index bd5c07c..11c80df 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,7 +57,7 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to. + *Server // Reference to the server we belong to. id int // Session ID number. conn net.Conn // Our network connection. remoteHost string // IP address of client. @@ -77,7 +77,7 @@ func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *S reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) return &Session{ - server: server, + Server: server, id: id, conn: conn, state: AUTHORIZATION, @@ -507,7 +507,7 @@ func (s *Session) sendMessageTop(msg storage.Message, lineCount int) { // Load the users mailbox func (s *Session) loadMailbox() { s.logger = s.logger.With().Str("mailbox", s.user).Logger() - m, err := s.server.store.GetMessages(s.user) + m, err := s.store.GetMessages(s.user) if err != nil { s.logger.Error().Msgf("Failed to load messages for %v: %v", s.user, err) } @@ -533,7 +533,7 @@ func (s *Session) processDeletes() { for i, msg := range s.messages { if !s.retain[i] { s.logger.Debug().Str("id", msg.ID()).Msg("Deleting message") - if err := s.server.store.RemoveMessage(s.user, msg.ID()); err != nil { + if err := s.store.RemoveMessage(s.user, msg.ID()); err != nil { s.logger.Warn().Str("id", msg.ID()).Err(err).Msg("Error deleting message") } } @@ -547,7 +547,7 @@ func (s *Session) enterState(state State) { // nextDeadline calculates the next read or write deadline based on configured timeout. func (s *Session) nextDeadline() time.Time { - return time.Now().Add(s.server.config.Timeout) + return time.Now().Add(s.config.Timeout) } // Send requested message, store errors in Session.sendError diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 9d4f238..0e30417 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -70,18 +70,18 @@ var commands = map[string]bool{ // Session holds the state of an SMTP session type Session struct { - server *Server - id int - conn net.Conn - remoteDomain string - remoteHost string - sendError error - state State - reader *bufio.Reader - from string - recipients []*policy.Recipient - logger zerolog.Logger // Session specific logger. - debug bool // Print network traffic to stdout. + *Server // Server this session belongs to. + id int // Session ID. + conn net.Conn // TCP connection. + remoteDomain string // Remote domain from HELO command. + remoteHost string // Remote host. + sendError error // Last network send error. + state State // Session state machine. + reader *bufio.Reader // Buffered reading for TCP conn. + from string // Sender from MAIL command. + recipients []*policy.Recipient // Recipients from RCPT commands. + logger zerolog.Logger // Session specific logger. + debug bool // Print network traffic to stdout. } // NewSession creates a new Session for the given connection @@ -89,7 +89,7 @@ func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *S reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) return &Session{ - server: server, + Server: server, id: id, conn: conn, state: GREET, @@ -244,7 +244,7 @@ func (s *Session) greetHandler(cmd string, arg string) { s.remoteDomain = domain s.send("250-Great, let's get this show on the road") s.send("250-8BITMIME") - s.send(fmt.Sprintf("250 SIZE %v", s.server.config.MaxMessageBytes)) + s.send(fmt.Sprintf("250 SIZE %v", s.config.MaxMessageBytes)) s.enterState(READY) default: s.ooSeq(cmd) @@ -296,7 +296,7 @@ func (s *Session) readyHandler(cmd string, arg string) { s.logger.Warn().Msgf("Unable to parse SIZE %q as an integer", args["SIZE"]) return } - if int(size) > s.server.config.MaxMessageBytes { + if int(size) > s.config.MaxMessageBytes { s.send("552 Max message size exceeded") s.logger.Warn().Msgf("Client wanted to send oversized message: %v", args["SIZE"]) return @@ -323,17 +323,17 @@ func (s *Session) mailHandler(cmd string, arg string) { } // This trim is probably too forgiving addr := strings.Trim(arg[3:], "<> ") - recip, err := s.server.addrPolicy.NewRecipient(addr) + recip, err := s.addrPolicy.NewRecipient(addr) if err != nil { s.send("501 Bad recipient address syntax") s.logger.Warn().Msgf("Bad address as RCPT arg: %q, %s", addr, err) return } - if len(s.recipients) >= s.server.config.MaxRecipients { + if len(s.recipients) >= s.config.MaxRecipients { s.logger.Warn().Msgf("Maximum limit of %v recipients reached", - s.server.config.MaxRecipients) + s.config.MaxRecipients) s.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", - s.server.config.MaxRecipients)) + s.config.MaxRecipients)) return } s.recipients = append(s.recipients, recip) @@ -381,10 +381,10 @@ func (s *Session) dataHandler() { if recip.ShouldStore() { // Generate Received header. prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - s.remoteDomain, s.remoteHost, s.server.config.Domain, recip.Address.Address, + s.remoteDomain, s.remoteHost, s.config.Domain, recip.Address.Address, tstamp) // Deliver message. - _, err := s.server.manager.Deliver( + _, err := s.manager.Deliver( recip, s.from, s.recipients, prefix, msgBuf.Bytes()) if err != nil { s.logger.Error().Msgf("delivery for %v: %v", recip.LocalPart, err) @@ -405,7 +405,7 @@ func (s *Session) dataHandler() { lineBuf = lineBuf[1:] } msgBuf.Write(lineBuf) - if msgBuf.Len() > s.server.config.MaxMessageBytes { + if msgBuf.Len() > s.config.MaxMessageBytes { s.send("552 Maximum message size exceeded") s.logger.Warn().Msgf("Max message size exceeded while in DATA") s.reset() @@ -420,12 +420,12 @@ func (s *Session) enterState(state State) { } func (s *Session) greet() { - s.send(fmt.Sprintf("220 %v Inbucket SMTP ready", s.server.config.Domain)) + s.send(fmt.Sprintf("220 %v Inbucket SMTP ready", s.config.Domain)) } // nextDeadline calculates the next read or write deadline based on configured timeout. func (s *Session) nextDeadline() time.Time { - return time.Now().Add(s.server.config.Timeout) + return time.Now().Add(s.config.Timeout) } // Send requested message, store errors in Session.sendError From e3be5362dc452217672ee478dcb7d4416b9f155e Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 1 Apr 2018 13:30:01 -0700 Subject: [PATCH 64/82] dev-start.sh: update TRACE to DEBUG --- etc/dev-start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/dev-start.sh b/etc/dev-start.sh index 6112cf3..4efd440 100755 --- a/etc/dev-start.sh +++ b/etc/dev-start.sh @@ -2,7 +2,7 @@ # dev-start.sh # description: Developer friendly Inbucket configuration -export INBUCKET_LOGLEVEL="TRACE" +export INBUCKET_LOGLEVEL="DEBUG" export INBUCKET_SMTP_DOMAINNOSTORE="bitbucket.local" export INBUCKET_WEB_TEMPLATECACHE="false" export INBUCKET_WEB_COOKIEAUTHKEY="not-secret" From cc5cd7f9c322e353588f4b761a9abfce44d6f0c3 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 31 Mar 2018 21:46:10 -0700 Subject: [PATCH 65/82] storage: Add Seen flag, tests for #58 --- CHANGELOG.md | 4 ++++ pkg/message/message.go | 6 ++++++ pkg/storage/file/fmessage.go | 6 ++++++ pkg/storage/file/fstore.go | 26 +++++++++++++++++++++++ pkg/storage/mem/message.go | 4 ++++ pkg/storage/mem/store.go | 11 ++++++++++ pkg/storage/storage.go | 2 ++ pkg/test/storage_suite.go | 41 ++++++++++++++++++++++++++++++++++++ 8 files changed, 100 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1872bd..127330b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,14 @@ This project adheres to [Semantic Versioning](http://semver.org/). page. - Debian `.deb` package generation to release process. - RedHat `.rpm` package generation to release process. +- Message seen flag in REST and Web UI so you can see which messages have + already been read. ### Changed - Massive refactor of back-end code. Inbucket should now be both easier and more enjoyable to work on. +- Changes to file storage format, will require pre-2.0 mail store directories to + be deleted. - Renamed `themes` directory to `ui` and eliminated the intermediate `bootstrap` directory. - Docker build: diff --git a/pkg/message/message.go b/pkg/message/message.go index 8bdd460..8c97cda 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -21,6 +21,7 @@ type Metadata struct { Date time.Time Subject string Size int64 + Seen bool } // Message holds both the metadata and content of a message. @@ -109,3 +110,8 @@ func (d *Delivery) Size() int64 { func (d *Delivery) Source() (io.ReadCloser, error) { return ioutil.NopCloser(d.Reader), nil } + +// Seen getter. +func (d *Delivery) Seen() bool { + return d.Meta.Seen +} diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 961df62..a7ed1e2 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -21,6 +21,7 @@ type Message struct { Fto []*mail.Address Fsubject string Fsize int64 + Fseen bool } // newMessage creates a new FileMessage object and sets the Date and ID fields. @@ -96,3 +97,8 @@ func (m *Message) Source() (reader io.ReadCloser, err error) { } return file, nil } + +// Seen returns the seen flag value. +func (m *Message) Seen() bool { + return m.Fseen +} diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 75ec3ba..b31ea0f 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -147,6 +147,32 @@ func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { return mb.getMessages() } +// MarkSeen flags the message as having been read. +func (fs *Store) MarkSeen(mailbox, id string) error { + mb, err := fs.mbox(mailbox) + if err != nil { + return err + } + mb.Lock() + defer mb.Unlock() + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return err + } + } + for _, m := range mb.messages { + if m.Fid == id { + if m.Fseen { + // Already marked seen. + return nil + } + m.Fseen = true + break + } + } + return mb.writeIndex() +} + // RemoveMessage deletes a message by ID from the specified mailbox. func (fs *Store) RemoveMessage(mailbox, id string) error { mb, err := fs.mbox(mailbox) diff --git a/pkg/storage/mem/message.go b/pkg/storage/mem/message.go index b5ca498..02ae503 100644 --- a/pkg/storage/mem/message.go +++ b/pkg/storage/mem/message.go @@ -21,6 +21,7 @@ type Message struct { date time.Time subject string source []byte + seen bool el *list.Element // This message in Store.messages } @@ -51,3 +52,6 @@ func (m *Message) Source() (io.ReadCloser, error) { // Size returns the message size in bytes. func (m *Message) Size() int64 { return int64(len(m.source)) } + +// Seen returns the message seen flag. +func (m *Message) Seen() bool { return m.seen } diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go index c37b241..16b094e 100644 --- a/pkg/storage/mem/store.go +++ b/pkg/storage/mem/store.go @@ -112,6 +112,17 @@ func (s *Store) GetMessages(mailbox string) (ms []storage.Message, err error) { return ms, err } +// MarkSeen marks a message as having been read. +func (s *Store) MarkSeen(mailbox, id string) error { + s.withMailbox(mailbox, true, func(mb *mbox) { + m := mb.messages[id] + if m != nil { + m.seen = true + } + }) + return nil +} + // PurgeMessages deletes the contents of a mailbox. func (s *Store) PurgeMessages(mailbox string) error { var messages map[string]*Message diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 67ef9fc..7cd40b3 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,6 +28,7 @@ type Store interface { AddMessage(message Message) (id string, err error) GetMessage(mailbox, id string) (Message, error) GetMessages(mailbox string) ([]Message, error) + MarkSeen(mailbox, id string) error PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error VisitMailboxes(f func([]Message) (cont bool)) error @@ -43,6 +44,7 @@ type Message interface { Subject() string Source() (io.ReadCloser, error) Size() int64 + Seen() bool } // FromConfig creates an instance of the Store based on the provided configuration. diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 9936145..b4135b0 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -28,6 +28,7 @@ func StoreSuite(t *testing.T, factory StoreFactory) { {"content", testContent, config.Storage{}}, {"delivery order", testDeliveryOrder, config.Storage{}}, {"size", testSize, config.Storage{}}, + {"seen", testSeen, config.Storage{}}, {"delete", testDelete, config.Storage{}}, {"purge", testPurge, config.Storage{}}, {"cap=10", testMsgCap, config.Storage{MailboxMsgCap: 10}}, @@ -65,6 +66,7 @@ func testMetadata(t *testing.T, store storage.Store) { To: to, Date: date, Subject: subject, + Seen: false, }, Reader: strings.NewReader(content), } @@ -107,6 +109,9 @@ func testMetadata(t *testing.T, store storage.Store) { if sm.Size() != int64(len(content)) { t.Errorf("got size %v, want: %v", sm.Size(), len(content)) } + if sm.Seen() { + t.Errorf("got seen %v, want: false", sm.Seen()) + } } // testContent generates some binary content and makes sure it is correctly retrieved. @@ -210,6 +215,42 @@ func testSize(t *testing.T, store storage.Store) { } } +// testSeen verifies a message can be marked as seen. +func testSeen(t *testing.T, store storage.Store) { + mailbox := "lisa" + id1, _ := DeliverToStore(t, store, mailbox, "whatever", time.Now()) + id2, _ := DeliverToStore(t, store, mailbox, "hello?", time.Now()) + // Confirm unseen. + msg, err := store.GetMessage(mailbox, id1) + if err != nil { + t.Fatal(err) + } + if msg.Seen() { + t.Errorf("got seen %v, want: false", msg.Seen()) + } + // Mark id1 seen. + err = store.MarkSeen(mailbox, id1) + if err != nil { + t.Fatal(err) + } + // Verify id1 seen. + msg, err = store.GetMessage(mailbox, id1) + if err != nil { + t.Fatal(err) + } + if !msg.Seen() { + t.Errorf("id1 got seen %v, want: true", msg.Seen()) + } + // Verify id2 still unseen. + msg, err = store.GetMessage(mailbox, id2) + if err != nil { + t.Fatal(err) + } + if msg.Seen() { + t.Errorf("id2 got seen %v, want: false", msg.Seen()) + } +} + // testDelete creates and deletes some messages. func testDelete(t *testing.T, store storage.Store) { mailbox := "fred" From dc02092cf6490580b530d2814bc62ce2484606bb Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 1 Apr 2018 11:13:33 -0700 Subject: [PATCH 66/82] rest: Implement MarkSeen for #58 - message: Add MarkSeen to Manager, StoreManager. - rest: Add PATCH for /mailbox/name/id. - rest: Add MailboxMarkSeenV1 handler. - rest: Add Seen to model. - rest: Update handlers to set Seen. - rest: Add doJSONBody func. --- pkg/message/manager.go | 6 +++ pkg/rest/apiv1_controller.go | 30 ++++++++++++ pkg/rest/apiv1_controller_test.go | 68 ++++++++++++++++++++++++++++ pkg/rest/client/apiv1_client.go | 16 +++++-- pkg/rest/client/apiv1_client_test.go | 35 +++++++++++++- pkg/rest/client/rest.go | 42 +++++++++++++---- pkg/rest/client/rest_test.go | 23 ++++++++-- pkg/rest/model/apiv1_model.go | 2 + pkg/rest/routes.go | 2 + pkg/rest/testutils_test.go | 26 +++++++++++ pkg/test/manager.go | 14 ++++++ 11 files changed, 247 insertions(+), 17 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 5e428e6..ff0a8a9 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -25,6 +25,7 @@ type Manager interface { ) (id string, err error) GetMetadata(mailbox string) ([]*Metadata, error) GetMessage(mailbox, id string) (*Message, error) + MarkSeen(mailbox, id string) error PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error SourceReader(mailbox, id string) (io.ReadCloser, error) @@ -124,6 +125,11 @@ func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { return &Message{Metadata: *header, env: env}, nil } +// MarkSeen marks the message as having been read. +func (s *StoreManager) MarkSeen(mailbox, id string) error { + return s.Store.MarkSeen(mailbox, id) +} + // PurgeMessages removes all messages from the specified mailbox. func (s *StoreManager) PurgeMessages(mailbox string) error { return s.Store.PurgeMessages(mailbox) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index adef527..ea71e43 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -7,6 +7,7 @@ import ( "crypto/md5" "encoding/hex" + "encoding/json" "strconv" "github.com/jhillyerd/inbucket/pkg/rest/model" @@ -37,6 +38,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( Subject: msg.Subject, Date: msg.Date, Size: msg.Size, + Seen: msg.Seen, } } return web.RenderJSON(w, jmessages) @@ -83,6 +85,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( Subject: msg.Subject, Date: msg.Date, Size: msg.Size, + Seen: msg.Seen, Header: msg.Header(), Body: &model.JSONMessageBodyV1{ Text: msg.Text(), @@ -92,6 +95,33 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( }) } +// MailboxMarkSeenV1 marks a message as read. +func MailboxMarkSeenV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { + // Don't have to validate these aren't empty, Gorilla returns 404 + id := ctx.Vars["id"] + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) + if err != nil { + return err + } + dec := json.NewDecoder(req.Body) + dm := model.JSONMessageHeaderV1{} + if err := dec.Decode(&dm); err != nil { + return fmt.Errorf("Failed to decode JSON: %v", err) + } + if dm.Seen { + err = ctx.Manager.MarkSeen(name, id) + if err == storage.ErrNotExist { + http.NotFound(w, req) + return nil + } + if err != nil { + // This doesn't indicate empty, likely an IO error + return fmt.Errorf("MarkSeen(%q) failed: %v", id, err) + } + } + return web.RenderJSON(w, "OK") +} + // MailboxPurgeV1 deletes all messages from a mailbox func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 734879e..cf64234 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -115,6 +115,7 @@ func TestRestMailboxList(t *testing.T) { decodedStringEquals(t, result, "[0]/subject", "subject 1") decodedStringEquals(t, result, "[0]/date", "2012-02-01T10:11:12.000000253-08:00") decodedNumberEquals(t, result, "[0]/size", 0) + decodedBoolEquals(t, result, "[0]/seen", false) decodedStringEquals(t, result, "[1]/mailbox", "good") decodedStringEquals(t, result, "[1]/id", "0002") decodedStringEquals(t, result, "[1]/from", "") @@ -122,6 +123,7 @@ func TestRestMailboxList(t *testing.T) { decodedStringEquals(t, result, "[1]/subject", "subject 2") decodedStringEquals(t, result, "[1]/date", "2012-07-01T10:11:12.000000253-07:00") decodedNumberEquals(t, result, "[1]/size", 0) + decodedBoolEquals(t, result, "[1]/seen", false) if t.Failed() { // Wait for handler to finish logging @@ -183,6 +185,7 @@ func TestRestMessage(t *testing.T) { To: []*mail.Address{{Name: "", Address: "to1@host"}}, Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, tzPST), + Seen: true, }, &enmime.Envelope{ Text: "This is some text", @@ -221,6 +224,7 @@ func TestRestMessage(t *testing.T) { decodedStringEquals(t, result, "subject", "subject 1") decodedStringEquals(t, result, "date", "2012-02-01T10:11:12.000000253-08:00") decodedNumberEquals(t, result, "size", 0) + decodedBoolEquals(t, result, "seen", true) decodedStringEquals(t, result, "body/text", "This is some text") decodedStringEquals(t, result, "body/html", "This is some HTML") decodedStringEquals(t, result, "header/To/[0]", "fred@fish.com") @@ -234,3 +238,67 @@ func TestRestMessage(t *testing.T) { _, _ = io.Copy(os.Stderr, logbuf) } } + +func TestRestMarkSeen(t *testing.T) { + mm := test.NewManager() + logbuf := setupWebServer(mm) + // Create some messages. + tzPDT := time.FixedZone("PDT", -7*3600) + tzPST := time.FixedZone("PST", -8*3600) + meta1 := message.Metadata{ + Mailbox: "good", + ID: "0001", + From: &mail.Address{Name: "", Address: "from1@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 1", + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, tzPST), + } + meta2 := message.Metadata{ + Mailbox: "good", + ID: "0002", + From: &mail.Address{Name: "", Address: "from2@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 2", + Date: time.Date(2012, 7, 1, 10, 11, 12, 253, tzPDT), + } + mm.AddMessage("good", &message.Message{Metadata: meta1}) + mm.AddMessage("good", &message.Message{Metadata: meta2}) + // Mark one read. + w, err := testRestPatch(baseURL+"/mailbox/good/0002", `{"seen":true}`) + expectCode := 200 + if err != nil { + t.Fatal(err) + } + if w.Code != expectCode { + t.Fatalf("Expected code %v, got %v", expectCode, w.Code) + } + // Get mailbox. + w, err = testRestGet(baseURL + "/mailbox/good") + expectCode = 200 + if err != nil { + t.Fatal(err) + } + if w.Code != expectCode { + t.Fatalf("Expected code %v, got %v", expectCode, w.Code) + } + // Check JSON. + dec := json.NewDecoder(w.Body) + var result []interface{} + if err := dec.Decode(&result); err != nil { + t.Errorf("Failed to decode JSON: %v", err) + } + if len(result) != 2 { + t.Fatalf("Expected 2 results, got %v", len(result)) + } + decodedStringEquals(t, result, "[0]/id", "0001") + decodedBoolEquals(t, result, "[0]/seen", false) + decodedStringEquals(t, result, "[1]/id", "0002") + decodedBoolEquals(t, result, "[1]/seen", true) + + if t.Failed() { + // Wait for handler to finish logging + time.Sleep(2 * time.Second) + // Dump buffered log data if there was a failure + _, _ = io.Copy(os.Stderr, logbuf) + } +} diff --git a/pkg/rest/client/apiv1_client.go b/pkg/rest/client/apiv1_client.go index ad62dcb..d88afce 100644 --- a/pkg/rest/client/apiv1_client.go +++ b/pkg/rest/client/apiv1_client.go @@ -58,10 +58,20 @@ func (c *Client) GetMessage(name, id string) (message *Message, err error) { return } +// MarkSeen marks the specified message as having been read. +func (c *Client) MarkSeen(name, id string) error { + uri := "/api/v1/mailbox/" + url.QueryEscape(name) + "/" + id + err := c.doJSON("PATCH", uri, nil) + if err != nil { + return err + } + return nil +} + // GetMessageSource returns the message source given a mailbox name and message ID. func (c *Client) GetMessageSource(name, id string) (*bytes.Buffer, error) { uri := "/api/v1/mailbox/" + url.QueryEscape(name) + "/" + id + "/source" - resp, err := c.do("GET", uri) + resp, err := c.do("GET", uri, nil) if err != nil { return nil, err } @@ -81,7 +91,7 @@ func (c *Client) GetMessageSource(name, id string) (*bytes.Buffer, error) { // DeleteMessage deletes a single message given the mailbox name and message ID. func (c *Client) DeleteMessage(name, id string) error { uri := "/api/v1/mailbox/" + url.QueryEscape(name) + "/" + id - resp, err := c.do("DELETE", uri) + resp, err := c.do("DELETE", uri, nil) if err != nil { return err } @@ -95,7 +105,7 @@ func (c *Client) DeleteMessage(name, id string) error { // PurgeMailbox deletes all messages in the given mailbox func (c *Client) PurgeMailbox(name string) error { uri := "/api/v1/mailbox/" + url.QueryEscape(name) - resp, err := c.do("DELETE", uri) + resp, err := c.do("DELETE", uri, nil) if err != nil { return err } diff --git a/pkg/rest/client/apiv1_client_test.go b/pkg/rest/client/apiv1_client_test.go index abce5ca..d3c8ca7 100644 --- a/pkg/rest/client/apiv1_client_test.go +++ b/pkg/rest/client/apiv1_client_test.go @@ -54,6 +54,32 @@ func TestClientV1GetMessage(t *testing.T) { } } +func TestClientV1MarkSeen(t *testing.T) { + var want, got string + + c, err := New(baseURLStr) + if err != nil { + t.Fatal(err) + } + mth := &mockHTTPClient{} + c.client = mth + + // Method under test + _ = c.MarkSeen("testbox", "20170107T224128-0000") + + want = "PATCH" + got = mth.req.Method + if got != want { + t.Errorf("req.Method == %q, want %q", got, want) + } + + want = baseURLStr + "/api/v1/mailbox/testbox/20170107T224128-0000" + got = mth.req.URL.String() + if got != want { + t.Errorf("req.URL == %q, want %q", got, want) + } +} + func TestClientV1GetMessageSource(t *testing.T) { var want, got string @@ -158,7 +184,8 @@ func TestClientV1MessageHeader(t *testing.T) { "from":"from1", "subject":"subject1", "date":"2017-01-01T00:00:00.000-07:00", - "size":100 + "size":100, + "seen":true } ]` @@ -216,6 +243,12 @@ func TestClientV1MessageHeader(t *testing.T) { t.Errorf("Subject == %q, want %q", got, want) } + wantb := true + gotb := header.Seen + if gotb != wantb { + t.Errorf("Seen == %v, want %v", gotb, wantb) + } + // Test MessageHeader.Delete() mth.body = "" err = header.Delete() diff --git a/pkg/rest/client/rest.go b/pkg/rest/client/rest.go index 718749d..1cefd55 100644 --- a/pkg/rest/client/rest.go +++ b/pkg/rest/client/rest.go @@ -1,8 +1,10 @@ package client import ( + "bytes" "encoding/json" "fmt" + "io" "net/http" "net/url" ) @@ -18,28 +20,48 @@ type restClient struct { baseURL *url.URL } -// do performs an HTTP request with this client and returns the response -func (c *restClient) do(method, uri string) (*http.Response, error) { +// do performs an HTTP request with this client and returns the response. +func (c *restClient) do(method, uri string, body []byte) (*http.Response, error) { rel, err := url.Parse(uri) if err != nil { return nil, err } - url := c.baseURL.ResolveReference(rel) - - // Build the request - req, err := http.NewRequest(method, url.String(), nil) + var r io.Reader + if body != nil { + r = bytes.NewReader(body) + } + req, err := http.NewRequest(method, url.String(), r) if err != nil { return nil, err } - - // Send the request return c.client.Do(req) } -// doGet performs an HTTP request with this client and marshalls the JSON response into v +// doJSON performs an HTTP request with this client and marshalls the JSON response into v. func (c *restClient) doJSON(method string, uri string, v interface{}) error { - resp, err := c.do(method, uri) + resp, err := c.do(method, uri, nil) + if err != nil { + return err + } + + defer func() { + _ = resp.Body.Close() + }() + if resp.StatusCode == http.StatusOK { + if v == nil { + return nil + } + // Decode response body + return json.NewDecoder(resp.Body).Decode(v) + } + + return fmt.Errorf("Unexpected HTTP response status %v: %s", resp.StatusCode, resp.Status) +} + +// doJSONBody performs an HTTP request with this client and marshalls the JSON response into v. +func (c *restClient) doJSONBody(method string, uri string, body []byte, v interface{}) error { + resp, err := c.do(method, uri, body) if err != nil { return err } diff --git a/pkg/rest/client/rest_test.go b/pkg/rest/client/rest_test.go index 3668578..c4d8f6e 100644 --- a/pkg/rest/client/rest_test.go +++ b/pkg/rest/client/rest_test.go @@ -35,17 +35,29 @@ func (m *mockHTTPClient) Do(req *http.Request) (resp *http.Response, err error) StatusCode: m.statusCode, Body: ioutil.NopCloser(bytes.NewBufferString(m.body)), } - return } +func (m *mockHTTPClient) ReqBody() []byte { + r, err := m.req.GetBody() + if err != nil { + return nil + } + body, err := ioutil.ReadAll(r) + if err != nil { + return nil + } + _ = r.Close() + return body +} + func TestDo(t *testing.T) { var want, got string - mth := &mockHTTPClient{} c := &restClient{mth, baseURL} + body := []byte("Test body") - _, err := c.do("POST", "/dopost") + _, err := c.do("POST", "/dopost", body) if err != nil { t.Fatal(err) } @@ -61,6 +73,11 @@ func TestDo(t *testing.T) { if got != want { t.Errorf("req.URL == %q, want %q", got, want) } + + b := mth.ReqBody() + if !bytes.Equal(b, body) { + t.Errorf("req.Body == %q, want %q", b, body) + } } func TestDoJSON(t *testing.T) { diff --git a/pkg/rest/model/apiv1_model.go b/pkg/rest/model/apiv1_model.go index 7e1e083..2b18e13 100644 --- a/pkg/rest/model/apiv1_model.go +++ b/pkg/rest/model/apiv1_model.go @@ -13,6 +13,7 @@ type JSONMessageHeaderV1 struct { Subject string `json:"subject"` Date time.Time `json:"date"` Size int64 `json:"size"` + Seen bool `json:"seen"` } // JSONMessageV1 contains the same data as the header plus a JSONMessageBody @@ -24,6 +25,7 @@ type JSONMessageV1 struct { Subject string `json:"subject"` Date time.Time `json:"date"` Size int64 `json:"size"` + Seen bool `json:"seen"` Body *JSONMessageBodyV1 `json:"body"` Header map[string][]string `json:"header"` Attachments []*JSONMessageAttachmentV1 `json:"attachments"` diff --git a/pkg/rest/routes.go b/pkg/rest/routes.go index fad722d..d622f3d 100644 --- a/pkg/rest/routes.go +++ b/pkg/rest/routes.go @@ -12,6 +12,8 @@ func SetupRoutes(r *mux.Router) { web.Handler(MailboxPurgeV1)).Name("MailboxPurgeV1").Methods("DELETE") r.Path("/api/v1/mailbox/{name}/{id}").Handler( web.Handler(MailboxShowV1)).Name("MailboxShowV1").Methods("GET") + r.Path("/api/v1/mailbox/{name}/{id}").Handler( + web.Handler(MailboxMarkSeenV1)).Name("MailboxMarkSeenV1").Methods("PATCH") r.Path("/api/v1/mailbox/{name}/{id}").Handler( web.Handler(MailboxDeleteV1)).Name("MailboxDeleteV1").Methods("DELETE") r.Path("/api/v1/mailbox/{name}/{id}/source").Handler( diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index b062083..67b7075 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -21,7 +21,17 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { if err != nil { return nil, err } + w := httptest.NewRecorder() + web.Router.ServeHTTP(w, req) + return w, nil +} +func testRestPatch(url string, body string) (*httptest.ResponseRecorder, error) { + req, err := http.NewRequest("PATCH", url, strings.NewReader(body)) + req.Header.Add("Accept", "application/json") + if err != nil { + return nil, err + } w := httptest.NewRecorder() web.Router.ServeHTTP(w, req) return w, nil @@ -46,6 +56,22 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { return buf } +func decodedBoolEquals(t *testing.T, json interface{}, path string, want bool) { + t.Helper() + els := strings.Split(path, "/") + val, msg := getDecodedPath(json, els...) + if msg != "" { + t.Errorf("JSON result%s", msg) + return + } + if got, ok := val.(bool); ok { + if got == want { + return + } + } + t.Errorf("JSON result/%s == %v (%T), want: %v", path, val, val, want) +} + func decodedNumberEquals(t *testing.T, json interface{}, path string, want float64) { t.Helper() els := strings.Split(path, "/") diff --git a/pkg/test/manager.go b/pkg/test/manager.go index bf8d9ad..f5053df 100644 --- a/pkg/test/manager.go +++ b/pkg/test/manager.go @@ -57,3 +57,17 @@ func (m *ManagerStub) GetMetadata(mailbox string) ([]*message.Metadata, error) { func (m *ManagerStub) MailboxForAddress(address string) (string, error) { return policy.ParseMailboxName(address) } + +// MarkSeen marks a message as having been read. +func (m *ManagerStub) MarkSeen(mailbox, id string) error { + if mailbox == "messageerr" { + return errors.New("internal error") + } + for _, msg := range m.mailboxes[mailbox] { + if msg.ID == id { + msg.Metadata.Seen = true + return nil + } + } + return storage.ErrNotExist +} From c695a2690de142e385be0f047ac8923dc4fa15db Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 1 Apr 2018 15:16:48 -0700 Subject: [PATCH 67/82] ui: Mark messages as seen after 1.5s for #58 Embolden subject font for unseen messages. --- etc/dev-start.sh | 2 +- pkg/message/manager.go | 4 ++++ ui/static/inbucket.css | 4 ++++ ui/static/mailbox.js | 36 ++++++++++++++++++++++++++++++--- ui/templates/mailbox/index.html | 5 +++-- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/etc/dev-start.sh b/etc/dev-start.sh index 4efd440..82b5cc9 100755 --- a/etc/dev-start.sh +++ b/etc/dev-start.sh @@ -8,7 +8,7 @@ export INBUCKET_WEB_TEMPLATECACHE="false" export INBUCKET_WEB_COOKIEAUTHKEY="not-secret" export INBUCKET_STORAGE_TYPE="file" export INBUCKET_STORAGE_PARAMS="path:/tmp/inbucket" -export INBUCKET_STORAGE_RETENTIONPERIOD="5m" +export INBUCKET_STORAGE_RETENTIONPERIOD="15m" if ! test -x ./inbucket; then echo "$PWD/inbucket not found/executable!" >&2 diff --git a/pkg/message/manager.go b/pkg/message/manager.go index ff0a8a9..35d0fcd 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -12,6 +12,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" + "github.com/rs/zerolog/log" ) // Manager is the interface controllers use to interact with messages. @@ -127,6 +128,8 @@ func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { // MarkSeen marks the message as having been read. func (s *StoreManager) MarkSeen(mailbox, id string) error { + log.Debug().Str("module", "manager").Str("mailbox", mailbox).Str("id", id). + Msg("Marking as seen") return s.Store.MarkSeen(mailbox, id) } @@ -164,5 +167,6 @@ func makeMetadata(m storage.Message) *Metadata { Date: m.Date(), Subject: m.Subject(), Size: m.Size(), + Seen: m.Seen(), } } diff --git a/ui/static/inbucket.css b/ui/static/inbucket.css index c895626..ece379e 100644 --- a/ui/static/inbucket.css +++ b/ui/static/inbucket.css @@ -29,6 +29,10 @@ body { font-size: 18px; } +.message-list-entry .unseen { + font-weight: bold; +} + .message-list-scroll { overflow-y: auto; } diff --git a/ui/static/mailbox.js b/ui/static/mailbox.js index 5a624c6..efe52d4 100644 --- a/ui/static/mailbox.js +++ b/ui/static/mailbox.js @@ -5,6 +5,7 @@ var messageListMargin = 275; var clipboard = null; var messageListScroll = false; var messageListData = null; +var seenDelay = null; // clearMessageSearch resets the message list search function clearMessageSearch() { @@ -55,8 +56,11 @@ function loadList() { url: '/api/v1/mailbox/' + mailbox, success: function(data) { messageListData = data.reverse(); + for (i=0; i