From 1ecc957aba4c0f1f5dd788e7a176cc34797eecee Mon Sep 17 00:00:00 2001 From: Alberto Bertogli Date: Fri, 11 Jan 2019 16:49:30 +0000 Subject: [PATCH] queue: Internationalized Delivery Status Notifications (DSN) Our non-delivery status notifications are quite simple today, but that makes it much more difficult to support internationalization and cross-language reporting. There is a standard for internationalized DSNs, RFC 6533 (which builds on top of the structured DSNs from RFC 3464). This patch changes our DSN messages to be based on those standards, so it is easier for MUAs to display reports according to the users' languages preferences. Note we still use message/rfc822 + 8bit to transmit the message, instead of message/global, for compatibility reasons. This seems to be more universally compatible, but the decision might be revisited in the future. See RFC 5335 (section 4.6 in particular). --- internal/queue/dsn.go | 83 +++++++++++++++++++++++++---- internal/queue/dsn_test.go | 55 +++++++++++++++---- test/t-05-null_address/content | 5 +- test/t-05-null_address/expected_dsr | 45 +++++++++++++--- test/t-05-null_address/sendmail.cmy | 5 +- 5 files changed, 160 insertions(+), 33 deletions(-) diff --git a/internal/queue/dsn.go b/internal/queue/dsn.go index 7ee2243..96783be 100644 --- a/internal/queue/dsn.go +++ b/internal/queue/dsn.go @@ -2,19 +2,23 @@ package queue import ( "bytes" + "net/mail" "text/template" "time" ) // Maximum length of the original message to include in the DSN. +// The receiver of the DSN might have a smaller message size than what we +// accepted, so we truncate to a value that should be large enough to be +// useful, but not problematic for modern deployments. const maxOrigMsgLen = 256 * 1024 // deliveryStatusNotification creates a delivery status notification (DSN) for // the given item, and puts it in the queue. // -// There is a standard, https://tools.ietf.org/html/rfc3464, although most -// MTAs seem to use a plain email and include an X-Failed-Recipients header. -// We're going with the latter for now, may extend it to the former later. +// References: +// - https://tools.ietf.org/html/rfc3464 (DSN) +// - https://tools.ietf.org/html/rfc6533 (Internationalized DSN) func deliveryStatusNotification(domainFrom string, item *Item) ([]byte, error) { info := dsnInfo{ OurDomain: domainFrom, @@ -44,11 +48,23 @@ func deliveryStatusNotification(domainFrom string, item *Item) ([]byte, error) { info.OriginalMessage = string(item.Data) } + info.OriginalMessageID = getMessageID(item.Data) + + info.Boundary = <-newID + buf := &bytes.Buffer{} err := dsnTemplate.Execute(buf, info) return buf.Bytes(), err } +func getMessageID(data []byte) string { + msg, err := mail.ReadMessage(bytes.NewBuffer(data)) + if err != nil { + return "" + } + return msg.Header.Get("Message-ID") +} + type dsnInfo struct { OurDomain string Destination string @@ -60,35 +76,80 @@ type dsnInfo struct { FailedRecipients []*Recipient PendingRecipients []*Recipient OriginalMessage string + + // Message-ID of the original message. + OriginalMessageID string + + // MIME boundary to use to form the message. + Boundary string } -var dsnTemplate = template.Must(template.New("dsn").Parse( - `From: Mail Delivery System +var dsnTemplate = template.Must( + template.New("dsn").Parse( + `From: Mail Delivery System To: <{{.Destination}}> Subject: Mail delivery failed: returning message to sender Message-ID: <{{.MessageID}}> Date: {{.Date}} +In-Reply-To: {{.OriginalMessageID}} +References: {{.OriginalMessageID}} X-Failed-Recipients: {{range .FailedTo}}{{.}}, {{end}} Auto-Submitted: auto-replied +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="{{.Boundary}}" -Delivery to the following recipient(s) failed permanently: + +--{{.Boundary}} +Content-Type: text/plain; charset="utf-8" +Content-Disposition: inline +Content-Description: Notification +Content-Transfer-Encoding: 8bit + +Delivery of your message to the following recipient(s) failed permanently: {{range .FailedTo -}} - {{.}} {{- end}} - ------ Technical details ----- -{{range .FailedRecipients}} +Technical details: +{{- range .FailedRecipients}} - "{{.Address}}" ({{.Type}}) failed permanently with error: {{.LastFailureMessage}} -{{end}} +{{- end}} {{- range .PendingRecipients}} - "{{.Address}}" ({{.Type}}) failed repeatedly and timed out, last error: {{.LastFailureMessage}} +{{- end}} + + +--{{.Boundary}} +Content-Type: message/global-delivery-status +Content-Description: Delivery Report +Content-Transfer-Encoding: 8bit + +Reporting-MTA: dns; {{.OurDomain}} + +{{range .FailedRecipients -}} +Original-Recipient: utf-8; {{.OriginalAddress}} +Final-Recipient: utf-8; {{.Address}} +Action: failed +Status: 5.0.0 +Diagnostic-Code: smtp; {{.LastFailureMessage}} +{{end}} +{{range .PendingRecipients -}} +Original-Recipient: utf-8; {{.OriginalAddress}} +Final-Recipient: utf-8; {{.Address}} +Action: failed +Status: 4.0.0 +Diagnostic-Code: smtp; {{.LastFailureMessage}} {{end}} ------ Original message ----- +--{{.Boundary}} +Content-Type: message/rfc822 +Content-Description: Undelivered Message +Content-Transfer-Encoding: 8bit {{.OriginalMessage}} +--{{.Boundary}}-- `)) diff --git a/internal/queue/dsn_test.go b/internal/queue/dsn_test.go index d4429fd..8e1cff4 100644 --- a/internal/queue/dsn_test.go +++ b/internal/queue/dsn_test.go @@ -12,12 +12,12 @@ func TestDSN(t *testing.T) { Message: Message{ ID: <-newID, From: "from@from.org", - To: []string{"toto@africa.org", "negra@sosa.org"}, + To: []string{"ñaca@africa.org", "negra@sosa.org"}, Rcpt: []*Recipient{ {"poe@rcpt", Recipient_EMAIL, Recipient_FAILED, - "oh! horror!", "toto@africa.org"}, + "oh! horror!", "ñaca@africa.org"}, {"newman@rcpt", Recipient_EMAIL, Recipient_PENDING, - "oh! the humanity!", "toto@africa.org"}, + "oh! the humanity!", "ñaca@africa.org"}, {"ant@rcpt", Recipient_EMAIL, Recipient_SENT, "", "negra@sosa.org"}, }, @@ -42,27 +42,60 @@ To: Subject: Mail delivery failed: returning message to sender Message-ID: Date: * -X-Failed-Recipients: toto@africa.org, +In-Reply-To: * +References: * +X-Failed-Recipients: ñaca@africa.org, Auto-Submitted: auto-replied - -Delivery to the following recipient(s) failed permanently: - - - toto@africa.org +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="???????????" ------ Technical details ----- +--??????????? +Content-Type: text/plain; charset="utf-8" +Content-Disposition: inline +Content-Description: Notification +Content-Transfer-Encoding: 8bit +Delivery of your message to the following recipient(s) failed permanently: + + - ñaca@africa.org + +Technical details: - "poe@rcpt" (EMAIL) failed permanently with error: oh! horror! - - "newman@rcpt" (EMAIL) failed repeatedly and timed out, last error: oh! the humanity! ------ Original message ----- +--??????????? +Content-Type: message/global-delivery-status +Content-Description: Delivery Report +Content-Transfer-Encoding: 8bit + +Reporting-MTA: dns; dsnDomain + +Original-Recipient: utf-8; ñaca@africa.org +Final-Recipient: utf-8; poe@rcpt +Action: failed +Status: 5.0.0 +Diagnostic-Code: smtp; oh! horror! + +Original-Recipient: utf-8; ñaca@africa.org +Final-Recipient: utf-8; newman@rcpt +Action: failed +Status: 4.0.0 +Diagnostic-Code: smtp; oh! the humanity! + + +--??????????? +Content-Type: message/rfc822 +Content-Description: Undelivered Message +Content-Transfer-Encoding: 8bit data ñaca +--???????????-- ` // flexibleEq compares two strings, supporting wildcards. diff --git a/test/t-05-null_address/content b/test/t-05-null_address/content index 911f0dc..d4e37cc 100644 --- a/test/t-05-null_address/content +++ b/test/t-05-null_address/content @@ -1,5 +1,6 @@ -From: Mailer daemon +From: Mailer daemon Subject: I've come to haunt you +Message-ID: -Muahahahaha +Ñañañañaña! diff --git a/test/t-05-null_address/expected_dsr b/test/t-05-null_address/expected_dsr index c5e83c2..b95611e 100644 --- a/test/t-05-null_address/expected_dsr +++ b/test/t-05-null_address/expected_dsr @@ -4,21 +4,49 @@ To: Subject: Mail delivery failed: returning message to sender Message-ID: * Date: * +In-Reply-To: +References: X-Failed-Recipients: fail@testserver, Auto-Submitted: auto-replied +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="???????????" -Delivery to the following recipient(s) failed permanently: + +--??????????? +Content-Type: text/plain; charset="utf-8" +Content-Disposition: inline +Content-Description: Notification +Content-Transfer-Encoding: 8bit + +Delivery of your message to the following recipient(s) failed permanently: - fail@testserver - ------ Technical details ----- - +Technical details: - "false" (PIPE) failed permanently with error: exit status 1 ------ Original message ----- +--??????????? +Content-Type: message/global-delivery-status +Content-Description: Delivery Report +Content-Transfer-Encoding: 8bit + +Reporting-MTA: dns; testserver + +Original-Recipient: utf-8; fail@testserver +Final-Recipient: utf-8; false +Action: failed +Status: 5.0.0 +Diagnostic-Code: smtp; exit status 1 + + + +--??????????? +Content-Type: message/rfc822 +Content-Description: Undelivered Message +Content-Transfer-Encoding: 8bit Received: from localhost by testserver (chasquid) with ESMTPSA @@ -26,9 +54,12 @@ Received: from localhost (over * ; * Date: * -From: Mailer daemon +From: Mailer daemon Subject: I've come to haunt you +Message-Id: -Muahahahaha +Ñañañañaña! + +--???????????-- diff --git a/test/t-05-null_address/sendmail.cmy b/test/t-05-null_address/sendmail.cmy index b413e8d..d3f33d8 100644 --- a/test/t-05-null_address/sendmail.cmy +++ b/test/t-05-null_address/sendmail.cmy @@ -10,10 +10,11 @@ c -> RCPT TO: user@testserver c <~ 250 c -> DATA c <~ 354 -c -> From: Mailer daemon +c -> From: Mailer daemon c -> Subject: I've come to haunt you +c -> Message-ID: c -> -c -> Muahahahaha +c -> Ñañañañaña! c -> c -> c -> .