Commit 69d158b1 authored by Barry Warsaw's avatar Barry Warsaw

Reorganize the Handler architecture to a pipeline architecture with plugins.

Now plugins can define additional handlers and the handlers can be organized
into named pipelines.  Modules are no longer the unit of a handler, now we use
classes so we can assert interface conformance.

The GLOBAL_PIPELINE is gone, replaced by the 'built-in' pipeline.  The
OWNER_PIPELINE is not yet replaced.

I still need a few more tests of the basic pipeline architecture, although the
individual handlers have pretty good coverage.

Added the IHandler and IPipeline interfaces.

Still broken, but not yet removed: Mailman/pipeline/moderate.py.
parent b36de8a6
......@@ -488,38 +488,6 @@ NNTP_REWRITE_DUPLICATE_HEADERS = [
# may wish to remove these headers by setting this to Yes.
REMOVE_DKIM_HEADERS = No
# All `normal' messages which are delivered to the entire list membership go
# through this pipeline of handler modules. Lists themselves can override the
# global pipeline by defining a `pipeline' attribute.
GLOBAL_PIPELINE = [
# These are the modules that do tasks common to all delivery paths.
'SpamDetect',
'Approve',
'Replybot',
'Moderate',
'Hold',
'MimeDel',
'Scrubber',
'Emergency',
'Tagger',
'CalcRecips',
'AvoidDuplicates',
'Cleanse',
'CleanseDKIM',
'CookHeaders',
# And now we send the message to the digest mbox file, and to the arch and
# news queues. Runners will provide further processing of the message,
# specific to those delivery paths.
'ToDigest',
'ToArchive',
'ToUsenet',
# Now we'll do a few extra things specific to the member delivery
# (outgoing) path, finally leaving the message in the outgoing queue.
'AfterDelivery',
'Acknowledge',
'ToOutgoing',
]
# This is the pipeline which messages sent to the -owner address go through
OWNER_PIPELINE = [
'SpamDetect',
......
......@@ -38,10 +38,10 @@ from Mailman.interfaces import LinkAction
def process(mlist, msg, msgdata, start_chain='built-in'):
"""Process the message through a chain.
:param start_chain: The name of the chain to start the processing with.
:param mlist: the IMailingList for this message.
:param msg: The Message object.
:param msgdata: The message metadata dictionary.
:param start_chain: The name of the chain to start the processing with.
"""
# Set up some bookkeeping.
chain_stack = []
......
# Copyright (C) 2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Pipeline processor."""
__metaclass__ = type
__all__ = [
'initialize',
'process',
]
from zope.interface import implements
from zope.interface.verify import verifyObject
from Mailman.app.plugins import get_plugins
from Mailman.configuration import config
from Mailman.i18n import _
from Mailman.interfaces import IHandler, IPipeline
def process(mlist, msg, msgdata, pipeline_name='built-in'):
"""Process the message through the given pipeline.
:param mlist: the IMailingList for this message.
:param msg: The Message object.
:param msgdata: The message metadata dictionary.
:param pipeline_name: The name of the pipeline to process through.
"""
pipeline = config.pipelines[pipeline_name]
for handler in pipeline:
handler.process(mlist, msg, msgdata)
class BuiltInPipeline:
"""The built-in pipeline."""
implements(IPipeline)
name = 'built-in'
description = _('The built-in pipeline.')
_default_handlers = (
'mimedel',
'scrubber',
'tagger',
'calculate-recipients',
'avoid-duplicates',
'cleanse',
'cleanse_dkim',
'cook_headers',
'to_digest',
'to_archive',
'to_usenet',
'after_delivery',
'acknowledge',
'to_outgoing',
)
def __init__(self):
self._handlers = []
for handler_name in self._default_handlers:
self._handler.append(config.handlers[handler_name])
def __iter__(self):
"""See `IPipeline`."""
for handler in self._handlers:
yield handler
def initialize():
"""Initialize the pipelines."""
# Find all handlers in the registered plugins.
for handler_finder in get_plugins('mailman.handlers'):
for handler_class in handler_finder():
handler = handler_class()
verifyObject(IHandler, handler)
assert handler.name not in config.handlers, (
'Duplicate handler "%s" found in %s' %
(handler.name, handler_finder))
config.handlers[handler.name] = handler
......@@ -179,6 +179,7 @@ class Configuration(object):
# Create the registry of rules and chains.
self.chains = {}
self.rules = {}
self.handlers = {}
def add_domain(self, email_host, url_host=None):
"""Add a virtual domain.
......
......@@ -67,8 +67,10 @@ def initialize_2(debug=False):
# circular imports.
from Mailman.app.chains import initialize as initialize_chains
from Mailman.app.rules import initialize as initialize_rules
from Mailman.app.pipelines import initialize as initialize_pipelines
initialize_rules()
initialize_chains()
initialize_pipelines()
def initialize(config_path=None, propagate_logs=False):
......
# Copyright (C) 2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Interface describing a pipeline handler."""
from zope.interface import Interface, Attribute
class IHandler(Interface):
"""A basic pipeline handler."""
name = Attribute('Handler name; must be unique.')
description = Attribute('A brief description of the handler.')
def process(mlist, msg, msgdata):
"""Run the handler.
:param mlist: The mailing list object.
:param msg: The message object.
:param msgdata: The message metadata.
"""
# Copyright (C) 2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Interface for describing pipelines."""
from zope.interface import Interface, Attribute
class IPipeline(Interface):
"""A pipeline of handlers."""
name = Attribute('Pipeline name; must be unique.')
description = Attribute('A brief description of this pipeline.')
def __iter__():
"""Iterate over all the handlers in this pipeline."""
# Copyright (C) 2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""The built in set of pipeline handlers."""
__metaclass__ = type
__all__ = ['initialize']
import os
import sys
from Mailman.interfaces import IHandler
def initialize():
"""Initialize the built-in handlers.
Rules are auto-discovered by searching for IHandler implementations in all
importable modules in this subpackage.
"""
# Find all rules found in all modules inside our package.
import Mailman.pipeline
here = os.path.dirname(Mailman.pipeline.__file__)
for filename in os.listdir(here):
basename, extension = os.path.splitext(filename)
if extension <> '.py':
continue
module_name = 'Mailman.pipeline.' + basename
__import__(module_name, fromlist='*')
module = sys.modules[module_name]
for name in getattr(module, '__all__', ()):
handler = getattr(module, name)
if IHandler.implementedBy(handler):
yield handler
......@@ -15,51 +15,66 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Send an acknowledgement of the successful post to the sender.
"""Send an acknowledgment of the successful post to the sender.
This only happens if the sender has set their AcknowledgePosts attribute.
This module must appear after the deliverer in the message pipeline in order
to send acks only after successful delivery.
"""
__metaclass__ = type
__all__ = ['Acknowledge']
from zope.interface import implements
from Mailman import Errors
from Mailman import Message
from Mailman import Utils
from Mailman.configuration import config
from Mailman.i18n import _
from Mailman.interfaces import IHandler
__i18n_templates__ = True
def process(mlist, msg, msgdata):
# Extract the sender's address and find them in the user database
sender = msgdata.get('original_sender', msg.get_sender())
member = mlist.members.get_member(sender)
if member is None:
return
ack = member.acknowledge_posts
if not ack:
return
# Okay, they want acknowledgement of their post. Give them their original
# subject. BAW: do we want to use the decoded header?
origsubj = msgdata.get('origsubj', msg.get('subject', _('(no subject)')))
# Get the user's preferred language
lang = msgdata.get('lang', member.preferred_language)
# Now get the acknowledgement template
realname = mlist.real_name
text = Utils.maketext(
'postack.txt',
{'subject' : Utils.oneline(origsubj, Utils.GetCharSet(lang)),
'listname' : realname,
'listinfo_url': mlist.script_url('listinfo'),
'optionsurl' : member.options_url,
}, lang=lang, mlist=mlist, raw=True)
# Craft the outgoing message, with all headers and attributes
# necessary for general delivery. Then enqueue it to the outgoing
# queue.
subject = _('$realname post acknowledgment')
usermsg = Message.UserNotification(sender, mlist.bounces_address,
subject, text, lang)
usermsg.send(mlist)
class Acknowledge:
"""Send an acknowledgment."""
implements(IHandler)
name = 'acknowledge'
description = _("""Send an acknowledgment of a posting.""")
def process(self, mlist, msg, msgdata):
"""See `IHandler`."""
# Extract the sender's address and find them in the user database
sender = msgdata.get('original_sender', msg.get_sender())
member = mlist.members.get_member(sender)
if member is None or not member.acknowledge_posts:
# Either the sender is not a member, in which case we can't know
# whether they want an acknowlegment or not, or they are a member
# who definitely does not want an acknowlegment.
return
# Okay, they are a member that wants an acknowledgment of their post.
# Give them their original subject. BAW: do we want to use the
# decoded header?
original_subject = msgdata.get(
'origsubj', msg.get('subject', _('(no subject)')))
# Get the user's preferred language.
lang = msgdata.get('lang', member.preferred_language)
# Now get the acknowledgement template.
realname = mlist.real_name
text = Utils.maketext(
'postack.txt',
{'subject' : Utils.oneline(original_subject,
Utils.GetCharSet(lang)),
'listname' : realname,
'listinfo_url': mlist.script_url('listinfo'),
'optionsurl' : member.options_url,
}, lang=lang, mlist=mlist, raw=True)
# Craft the outgoing message, with all headers and attributes
# necessary for general delivery. Then enqueue it to the outgoing
# queue.
subject = _('$realname post acknowledgment')
usermsg = Message.UserNotification(sender, mlist.bounces_address,
subject, text, lang)
usermsg.send(mlist)
......@@ -15,15 +15,30 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""Perform some bookkeeping after a successful post.
"""Perform some bookkeeping after a successful post."""
__metaclass__ = type
__all__ = ['AfterDelivery']
This module must appear after the delivery module in the message pipeline.
"""
import datetime
from zope.interface import implements
from Mailman.i18n import _
from Mailman.interfaces import IHandler
def process(mlist, msg, msgdata):
mlist.last_post_time = datetime.datetime.now()
mlist.post_id += 1
class AfterDelivery:
"""Perform some bookkeeping after a successful post."""
implements(IHandler)
name = 'after-delivery'
description = _('Perform some bookkeeping after a successful post.')
def process(self, mlist, msg, msgdata):
"""See `IHander`."""
mlist.last_post_time = datetime.datetime.now()
mlist.post_id += 1
......@@ -23,71 +23,91 @@ has already received a copy, we either drop the message, add a duplicate
warning header, or pass it through, depending on the user's preferences.
"""
__metaclass__ = type
__all__ = ['AvoidDuplicates']
from email.Utils import getaddresses, formataddr
from zope.interface import implements
from Mailman.configuration import config
from Mailman.i18n import _
from Mailman.interfaces import IHandler
COMMASPACE = ', '
def process(mlist, msg, msgdata):
recips = msgdata.get('recips')
# Short circuit
if not recips:
return
# Seed this set with addresses we don't care about dup avoiding.
listaddrs = set((mlist.posting_address,
mlist.bounces_address,
mlist.owner_address,
mlist.request_address))
explicit_recips = listaddrs.copy()
# Figure out the set of explicit recipients
cc_addresses = {}
for header in ('to', 'cc', 'resent-to', 'resent-cc'):
addrs = getaddresses(msg.get_all(header, []))
header_addresses = dict((addr, formataddr((name, addr)))
for name, addr in addrs
if addr)
if header == 'cc':
# Yes, it's possible that an address is mentioned in multiple CC
# headers using different names. In that case, the last real name
# will win, but that doesn't seem like such a big deal. Besides,
# how else would you chose?
cc_addresses.update(header_addresses)
# Ignore the list addresses for purposes of dup avoidance.
explicit_recips |= set(header_addresses)
# Now strip out the list addresses
explicit_recips -= listaddrs
if not explicit_recips:
# No one was explicitly addressed, so we can't do any dup collapsing
return
newrecips = set()
for r in recips:
# If this recipient is explicitly addressed...
if r in explicit_recips:
send_duplicate = True
# If the member wants to receive duplicates, or if the recipient
# is not a member at all, they will get a copy.
# header.
member = mlist.members.get_member(r)
if member and not member.receive_list_copy:
send_duplicate = False
# We'll send a duplicate unless the user doesn't wish it. If
# personalization is enabled, the add-dupe-header flag will add a
# X-Mailman-Duplicate: yes header for this user's message.
if send_duplicate:
msgdata.setdefault('add-dup-header', set()).add(r)
class AvoidDuplicates:
"""If the user wishes it, do not send duplicates of the same message."""
implements(IHandler)
name = 'avoid-duplicates'
description = _('Suppress some duplicates of the same message.')
def process(self, mlist, msg, msgdata):
"""See `IHandler`."""
recips = msgdata.get('recips')
# Short circuit
if not recips:
return
# Seed this set with addresses we don't care about dup avoiding.
listaddrs = set((mlist.posting_address,
mlist.bounces_address,
mlist.owner_address,
mlist.request_address))
explicit_recips = listaddrs.copy()
# Figure out the set of explicit recipients.
cc_addresses = {}
for header in ('to', 'cc', 'resent-to', 'resent-cc'):
addrs = getaddresses(msg.get_all(header, []))
header_addresses = dict((addr, formataddr((name, addr)))
for name, addr in addrs
if addr)
if header == 'cc':
# Yes, it's possible that an address is mentioned in multiple
# CC headers using different names. In that case, the last
# real name will win, but that doesn't seem like such a big
# deal. Besides, how else would you chose?
cc_addresses.update(header_addresses)
# Ignore the list addresses for purposes of dup avoidance.
explicit_recips |= set(header_addresses)
# Now strip out the list addresses.
explicit_recips -= listaddrs
if not explicit_recips:
# No one was explicitly addressed, so we can't do any dup
# collapsing
return
newrecips = set()
for r in recips:
# If this recipient is explicitly addressed...
if r in explicit_recips:
send_duplicate = True
# If the member wants to receive duplicates, or if the
# recipient is not a member at all, they will get a copy.
# header.
member = mlist.members.get_member(r)
if member and not member.receive_list_copy:
send_duplicate = False
# We'll send a duplicate unless the user doesn't wish it. If
# personalization is enabled, the add-dupe-header flag will
# add a X-Mailman-Duplicate: yes header for this user's
# message.
if send_duplicate:
msgdata.setdefault('add-dup-header', set()).add(r)
newrecips.add(r)
elif r in cc_addresses:
del cc_addresses[r]
else:
# Otherwise, this is the first time they've been in the recips
# list. Add them to the newrecips list and flag them as
# having received this message.
newrecips.add(r)
elif r in cc_addresses:
del cc_addresses[r]
else:
# Otherwise, this is the first time they've been in the recips
# list. Add them to the newrecips list and flag them as having
# received this message.
newrecips.add(r)
# Set the new list of recipients. XXX recips should always be a set.
msgdata['recips'] = list(newrecips)
# RFC 2822 specifies zero or one CC header
if cc_addresses:
del msg['cc']
msg['CC'] = COMMASPACE.join(cc_addresses.values())
# Set the new list of recipients. XXX recips should always be a set.
msgdata['recips'] = list(newrecips)
# RFC 2822 specifies zero or one CC header
if cc_addresses:
del msg['cc']
msg['CC'] = COMMASPACE.join(cc_addresses.values())
......@@ -23,62 +23,78 @@ on the `recips' attribute of the message. This attribute is used by the
SendmailDeliver and BulkDeliver modules.
"""
__metaclass__ = type
__all__ = ['CalculateRecipients']
from zope.interface import implements
from Mailman import Errors
from Mailman import Message
from Mailman import Utils
from Mailman.configuration import config
from Mailman.i18n import _
from Mailman.interfaces import DeliveryStatus
from Mailman.interfaces import DeliveryStatus, IHandler
def process(mlist, msg, msgdata):
# Short circuit if we've already calculated the recipients list,
# regardless of whether the list is empty or not.
if 'recips' in msgdata:
return
# Should the original sender should be included in the recipients list?
include_sender = True
sender = msg.get_sender()
member = mlist.members.get_member(sender)
if member and not member.receive_own_postings:
include_sender = False
# Support for urgent messages, which bypasses digests and disabled
# delivery and forces an immediate delivery to all members Right Now. We
# are specifically /not/ allowing the site admins password to work here
# because we want to discourage the practice of sending the site admin
# password through email in the clear. (see also Approve.py)
missing = []
password = msg.get('urgent', missing)
if password is not missing:
if mlist.Authenticate((config.AuthListModerator,
config.AuthListAdmin),
password):
recips = mlist.getMemberCPAddresses(mlist.getRegularMemberKeys() +
mlist.getDigestMemberKeys())
msgdata['recips'] = recips
class CalculateRecipients:
"""Calculate the regular (i.e. non-digest) recipients of the message."""
implements(IHandler)
name = 'calculate-recipients'
description = _('Calculate the regular recipients of the message.')
def process(self, mlist, msg, msgdata):
# Short circuit if we've already calculated the recipients list,
# regardless of whether the list is empty or not.
if 'recips' in msgdata:
return
else:
# Bad Urgent: password, so reject it instead of passing it on. I
# think it's better that the sender know they screwed up than to
# deliver it normally.
realname = mlist.real_name
text = _("""\
# Should the original sender should be included in the recipients list?
include_sender = True
sender = msg.get_sender()
member = mlist.members.get_member(sender)
if member and not member.receive_own_postings:
include_sender = False
# Support for urgent messages, which bypasses digests and disabled
# delivery and forces an immediate delivery to all members Right Now.
# We are specifically /not/ allowing the site admins password to work
# here because we want to discourage the practice of sending the site
# admin password through email in the clear. (see also Approve.py)
#
# XXX This is broken.
missing = object()
password = msg.get('urgent', missing)
if password is not missing:
if mlist.Authenticate((config.AuthListModerator,
config.AuthListAdmin),
password):
recips = mlist.getMemberCPAddresses(
mlist.getRegularMemberKeys() +
mlist.getDigestMemberKeys())
msgdata['recips'] = recips
return
else:
# Bad Urgent: password, so reject it instead of passing it on.
# I think it's better that the sender know they screwed up
# than to deliver it normally.
realname = mlist.real_name
text = _("""\
Your urgent message to the %(realname)s mailing list was not authorized for
delivery. The original message as received by Mailman is attached.
""")
raise Errors.RejectMessage, Utils.wrap(text)
# Calculate the regular recipients of the message
recips = set(member.address.address
for member in mlist.regular_members.members
if member.delivery_status == DeliveryStatus.enabled)
# Remove the sender if they don't want to receive their own posts
if not include_sender and member.address.address in recips:
recips.remove(member.address.address)
# Handle topic classifications
do_topic_filters(mlist, msg, msgdata, recips)
# Bookkeeping
msgdata['recips'] = recips
raise Errors.RejectMessage, Utils.wrap(text)
# Calculate the regular recipients of the message
recips = set(member.address.address
for member in mlist.regular_members.members
if member.delivery_status == DeliveryStatus.enabled)
# Remove the sender if they don't want to receive their own posts
if not include_sender and member.address.address in recips:
recips.remove(member.address.address)
# Handle topic classifications
do_topic_filters(mlist, msg, msgdata, recips)
# Bookkeeping
msgdata['recips'] = recips
......
......@@ -17,42 +17,55 @@
"""Cleanse certain headers from all messages."""
__metaclass__ = type
__all__ = ['Cleanse']