Verified Commit e1d20b31 authored by Mark Sapiro's avatar Mark Sapiro
Browse files

Implement the attachment of the DSN to bounce probes.

parent a063b8f5
Pipeline #340946203 failed with stage
......@@ -216,6 +216,9 @@ Message-ID: <first>
self.assertEqual(message.get_content_type(), 'multipart/mixed')
self.assertTrue(message.is_multipart())
self.assertEqual(len(message.get_payload()), 2)
# Check that the second part is the DSN
part_content = message.get_payload(1).get_payload(0).as_string()
self.assertEqual(part_content, self._msg.as_string())
def test_probe_sends_one_message(self):
# send_probe() places one message in the virgin queue. We start out
......
......@@ -39,6 +39,7 @@ Bugs
* Attempts to get a message from the message store with a missing file are
now handled. (Closes #877)
* The task runner no longer prematurely deletes saved DSNs. (Closes #878)
* Bounce probe messages now contain the DSN as advertised. (Closes #880)
* The avoid_duplicates handler properly handles headers that are returned as
email.header.Header instances rather than strings. (Closeds #881)
* The mta.deliver module properly handles headers that are returned as
......
......@@ -20,8 +20,9 @@
import logging
import datetime
from lazr.config import as_boolean
from mailman.app.bounces import send_probe
from email.utils import make_msgid
from lazr.config import as_boolean, as_timedelta
from mailman.app.bounces import PENDABLE_LIFETIME, _ProbePendable, send_probe
from mailman.app.membership import delete_member
from mailman.app.notifications import (
send_admin_disable_notice, send_admin_increment_notice,
......@@ -34,6 +35,8 @@ from mailman.interfaces.bounce import (
BounceContext, IBounceEvent, IBounceProcessor, InvalidBounceEvent)
from mailman.interfaces.listmanager import IListManager
from mailman.interfaces.member import DeliveryStatus, IMembershipManager
from mailman.interfaces.messages import IMessageStore
from mailman.interfaces.pending import IPendings
from mailman.utilities.datetime import now
from public import public
from sqlalchemy import Boolean, Column, DateTime, Integer
......@@ -77,6 +80,20 @@ class BounceProcessor:
@dbconnection
def register(self, store, mlist, email, msg, where=None):
"""See `IBounceProcessor`."""
# Save the DSN in the message store for notices. It doesn't
# matter if we save it more than once. Only one copy will be
# saved, but ensure it has a Message-ID so we can retreive it.
if msg.get('message-id') is None:
msg['Message-ID'] = make_msgid # pragma: nocover
getUtility(IMessageStore).add(msg)
# We also need to pend a token for this or the message will be
# removed as an orphan by the task runner. We don't need much
# from this. We pend the msgid as _mod_message_id for the
# task runner.
pendable = _ProbePendable(
_mod_message_id=msg.get('message-id'))
getUtility(IPendings).add(
pendable, lifetime=as_timedelta(PENDABLE_LIFETIME))
event = BounceEvent(mlist.list_id, email, msg, where)
store.add(event)
return event
......@@ -200,8 +217,10 @@ class BounceProcessor:
if member.bounce_score >= mlist.bounce_score_threshold:
# Save bounce_score because sending probe resets it.
saved_bounce_score = member.bounce_score
# Try to get the dsn from the message store. It should be there.
msg = getUtility(IMessageStore).get_message_by_id(event.message_id)
if as_boolean(config.mta.verp_probes):
send_probe(member, message_id=event.message_id)
send_probe(member, msg=msg, message_id=event.message_id)
action = 'sending probe'
else:
self._disable_delivery(mlist, member, event)
......
......@@ -213,7 +213,7 @@ Next bounce event for anne should trigger a probe which resets bounce_score:
>>> msg = items[0].msg
>>> print(msg.as_string())
Subject: Test mailing list probe message
From: test-bounces+0000000000000000000000000000000000000001@example.com
From: test-bounces+...@example.com
To: anne@example.com
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="..."
......@@ -242,8 +242,13 @@ Next bounce event for anne should trigger a probe which resets bounce_score:
test-owner@example.com
<BLANKLINE>
...
Content-Type: message/rfc822
MIME-Version: 1.0
<BLANKLINE>
From: mail-daemon@example.org
To: test-bounces@example.com
Message-ID: <second>
...
When such a probe bounces, their delivery is then suspended immediately:
......
......@@ -146,8 +146,9 @@ def deliver(mlist, msg, msgdata):
# so the logic below works.
#
if code >= 500 and code != 552:
# A permanent failure
permanent_failures.append(recipient)
# A permanent failure. Keep the code and message for a fake DSN.
permanent_failures.append(
(recipient, code, smtp_message)) # pragma: nocover
else:
# Deal with persistent transient failures by queuing them up for
# future delivery. TBD: this could generate lots of log entries!
......
......@@ -19,16 +19,11 @@
import logging
from email.utils import make_msgid
from flufl.bounce import all_failures
from lazr.config import as_timedelta
from mailman.app.bounces import (
PENDABLE_LIFETIME, ProbeVERP, StandardVERP, _ProbePendable, maybe_forward)
from mailman.app.bounces import ProbeVERP, StandardVERP, maybe_forward
from mailman.core.runner import Runner
from mailman.interfaces.bounce import (
BounceContext, IBounceProcessor, InvalidBounceEvent)
from mailman.interfaces.messages import IMessageStore
from mailman.interfaces.pending import IPendings
from public import public
from zope.component import getUtility
......@@ -91,20 +86,6 @@ class BounceRunner(Runner):
log.exception('Ignoring non-UTF-8 encoded '
'address: {}'.format(address))
continue
# Save the DSN in the message store for notices. It doesn't
# matter if we save it more than once. Only one copy will be
# saved, but ensure it has a Message-ID so we can retreive it.
if msg.get('message-id') is None:
msg['Message-ID'] = make_msgid # pragma: nocover
getUtility(IMessageStore).add(msg)
# We also need to pend a token for this or the message will be
# removed as an orphan by the task runner. We don't need much
# from this. We pend the msgid as _mod_message_id for the
# task runner.
pendable = _ProbePendable(
_mod_message_id=msg.get('message-id'))
getUtility(IPendings).add(
pendable, lifetime=as_timedelta(PENDABLE_LIFETIME))
self._processor.register(mlist, address, msg, context)
else:
log.info('Bounce message w/no discernable addresses: %s',
......
......@@ -21,9 +21,11 @@ import socket
import logging
from datetime import datetime
from email.utils import formatdate, make_msgid
from lazr.config import as_boolean, as_timedelta
from mailman.config import config
from mailman.core.runner import Runner
from mailman.email.message import Message
from mailman.interfaces.bounce import BounceContext, IBounceProcessor
from mailman.interfaces.mailinglist import Personalization
from mailman.interfaces.mta import SomeRecipientsFailed
......@@ -59,6 +61,22 @@ class OutgoingRunner(Runner):
self._logged = False
self._retryq = config.switchboards['retry']
def _fake_dsn(self, recipient, code, smtp_message):
# Craft a fake DSN for SMTP permanent failures.
msg = Message()
msg['From'] = 'Mailman <mailman@example.com>'
msg['To'] = 'Mailman Bounces <mailman-bounces@example.com>'
msg['Subject'] = 'SMTP Delivery Failure'
msg['Message-ID'] = make_msgid()
msg['Date'] = formatdate(localtime=True)
msg.set_payload("""\
Mail to {} failed at outgoing SMTP
Error code: {}
Error message: {}
""".format(recipient, code, smtp_message))
return msg
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0))
......@@ -126,6 +144,7 @@ class OutgoingRunner(Runner):
# The UUID had to be pended as a unicode.
member = getUtility(ISubscriptionService).get_member(
UUID(hex=pended['member_id']))
msg = self._fake_dsn(*error.permanent_failures[0])
processor.register(
mlist, member.address.email, msg,
BounceContext.probe)
......@@ -133,7 +152,8 @@ class OutgoingRunner(Runner):
# Delivery failed at SMTP time for some or all of the
# recipients. Permanent failures are registered as bounces,
# but temporary failures are retried for later.
for email in error.permanent_failures:
for email, code, smtp_message in error.permanent_failures:
msg = self._fake_dsn(email, code, smtp_message)
processor.register(mlist, email, msg, BounceContext.normal)
# Move temporary failures to the qfiles/retry queue which will
# occasionally move them back here for another shot at
......
......@@ -30,6 +30,7 @@ from mailman.config import config
from mailman.interfaces.bounce import BounceContext, IBounceProcessor
from mailman.interfaces.mailinglist import Personalization
from mailman.interfaces.member import MemberRole
from mailman.interfaces.messages import IMessageStore
from mailman.interfaces.mta import SomeRecipientsFailed
from mailman.interfaces.pending import IPendings
from mailman.interfaces.usermanager import IUserManager
......@@ -307,12 +308,12 @@ Message-Id: <first>
def test_probe_failure(self):
# When a probe message fails during SMTP, a bounce event is recorded
# with the proper bounce context.
# with the proper bounce context and a fake DSN is recorded.
anne = getUtility(IUserManager).create_address('anne@example.com')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
msgdata = dict(probe_token=token)
permanent_failures.append('anne@example.com')
permanent_failures.append(('anne@example.com', 500, 'Failure'))
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
......@@ -321,9 +322,14 @@ Message-Id: <first>
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, 'anne@example.com')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
self.assertEqual(event.context, BounceContext.probe)
self.assertFalse(event.processed)
# We can't say anything about the Message-ID because it's generated,
# But we can get the message.
msg = getUtility(IMessageStore).get_message_by_id(event.message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 500', msg.as_string())
self.assertIn('Error message: Failure', msg.as_string())
def test_confirmed_probe_failure(self):
# This time, a probe also fails, but for some reason the probe token
......@@ -333,7 +339,7 @@ Message-Id: <first>
token = send_probe(member, self._msg)
getUtility(IPendings).confirm(token)
msgdata = dict(probe_token=token)
permanent_failures.append('anne@example.com')
permanent_failures.append(('anne@example.com', 500, 'Failure'))
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
......@@ -355,18 +361,23 @@ Message-Id: <first>
def test_one_permanent_failure(self):
# Normal (i.e. non-probe) permanent failures just get registered.
permanent_failures.append('anne@example.com')
permanent_failures.append(('anne@example.com', 500, 'Failure'))
self._outq.enqueue(self._msg, {}, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, 'anne@example.com')
self.assertEqual(events[0].context, BounceContext.normal)
# Check the fake DSN we created.
msg = getUtility(IMessageStore).get_message_by_id(events[0].message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 500', msg.as_string())
self.assertIn('Error message: Failure', msg.as_string())
def test_two_permanent_failures(self):
# Two normal (i.e. non-probe) permanent failures just get registered.
permanent_failures.append('anne@example.com')
permanent_failures.append('bart@example.com')
permanent_failures.append(('anne@example.com', 500, 'Failure'))
permanent_failures.append(('bart@example.com', 501, 'Another Failure'))
self._outq.enqueue(self._msg, {}, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
......@@ -375,6 +386,15 @@ Message-Id: <first>
self.assertEqual(events[0].context, BounceContext.normal)
self.assertEqual(events[1].email, 'bart@example.com')
self.assertEqual(events[1].context, BounceContext.normal)
# Check the fake DSNs we created.
msg = getUtility(IMessageStore).get_message_by_id(events[0].message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 500', msg.as_string())
self.assertIn('Error message: Failure', msg.as_string())
msg = getUtility(IMessageStore).get_message_by_id(events[1].message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 501', msg.as_string())
self.assertIn('Error message: Another Failure', msg.as_string())
def test_one_temporary_failure(self):
# The first time there are temporary failures, the message just gets
......@@ -414,8 +434,8 @@ Message-Id: <first>
def test_mixed_failures(self):
# Some temporary and some permanent failures.
permanent_failures.append('elle@example.com')
permanent_failures.append('fred@example.com')
permanent_failures.append(('elle@example.com', 500, 'Failure'))
permanent_failures.append(('fred@example.com', 501, 'Another Failure'))
temporary_failures.append('gwen@example.com')
temporary_failures.append('herb@example.com')
self._outq.enqueue(self._msg, {}, listid='test.example.com')
......@@ -427,6 +447,15 @@ Message-Id: <first>
self.assertEqual(events[0].context, BounceContext.normal)
self.assertEqual(events[1].email, 'fred@example.com')
self.assertEqual(events[1].context, BounceContext.normal)
# Check the fake DSNs we created.
msg = getUtility(IMessageStore).get_message_by_id(events[0].message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 500', msg.as_string())
self.assertIn('Error message: Failure', msg.as_string())
msg = getUtility(IMessageStore).get_message_by_id(events[1].message_id)
self.assertEqual('SMTP Delivery Failure', msg.get('subject'))
self.assertIn('Error code: 501', msg.as_string())
self.assertIn('Error message: Another Failure', msg.as_string())
# Let's look at the temporary failures.
items = get_queue_messages('retry', expected_count=1)
self.assertEqual(items[0].msgdata['recipients'],
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment