Commit b18f632f authored by bwarsaw's avatar bwarsaw

Merge exp-elixir-branch to trunk. There is enough working to make me feel

confident the Elixir branch is ready to become mainline.  Also, fewer branches
makes for an easier migration to a dvcs.

Don't expect much of the old test suite to work, or even for much of the old
functionality to work.  The changes here are disruptive enough to break higher
level parts of Mailman.  But that's okay because I am slowly building up a new
and improved test suite, which will lead to a functional system again.

For now, only the doctests in Mailman/docs (and their related test harnesses)
will pass, but they all do pass.  Note that Mailman/docs serve as system
documentation first and unit tests second.  You should be able to read the
doctest files to understand the underlying data model.

Other changes included in this merge:

- Added the Mailman.ext extension package.
- zope.interfaces uses to describe major components
- SQLAlchemy/Elixir used as the database model
- Top level doinstall target renamed to justinstall
- 3rd-party packages are now installed in pythonlib/lib/python to be more
  compliant with distutils standards.  This allows us to use just --home
  instead of all the --install-* options.
- No longer need to include the email package or pysqlite, as Python 2.5 is
  required (and comes with both packages).
- munepy package is included, for Python enums
- IRosterSets are added as a way to manage a collection of IRosters.  Roster
  sets are named so that we can maintain the indirection between mailing lists
  and rosters, where the two are maintained in different storages.
- IMailingListRosters: remove_*_roster() -> delete_*_roster()
- Remove IMember interface.
- Utils.list_names() -> config.list_manager.names
- fqdn_listname() takes an optional hostname argument.
- Added a bunch of new exceptions used throughout the new interfaces.
- Make LockFile a context manager for use with the 'with' statement.
parent 5ff792b1
......@@ -182,10 +182,7 @@ def admin_overview(msg=''):
bgcolor=mm_cfg.WEB_HEADER_COLOR)
# Skip any mailing list that isn't advertised.
advertised = []
listnames = list(Utils.list_names())
listnames.sort()
for name in listnames:
for name in sorted(config.list_manager.names):
mlist = MailList.MailList(name, lock=False)
if mlist.advertised:
if hostname not in mlist.web_page_url:
......
......@@ -82,10 +82,7 @@ def listinfo_overview(msg=''):
# Skip any mailing lists that isn't advertised.
advertised = []
listnames = list(Utils.list_names())
listnames.sort()
for name in listnames:
for name in sorted(config.list_manager.names):
mlist = MailList.MailList(name, lock=False)
if mlist.advertised:
if hostname not in mlist.web_page_url:
......
......@@ -895,7 +895,7 @@ def loginpage(mlist, doc, user, lang):
def lists_of_member(mlist, user):
hostname = mlist.host_name
onlists = []
for listname in Utils.list_names():
for listname in config.list_manager.names:
# The current list will always handle things in the mainline
if listname == mlist.internal_name():
continue
......
......@@ -21,8 +21,8 @@
"""
from Mailman import mm_cfg
from Mailman import Utils
from Mailman.MailList import MailList
from Mailman.configuration import config
from Mailman.i18n import _
......@@ -43,10 +43,8 @@ def process(res, args):
return STOP
hostname = mlist.host_name
res.results.append(_('Public mailing lists at %(hostname)s:'))
lists = Utils.list_names()
lists.sort()
i = 1
for listname in lists:
for listname in sorted(config.list_manager.names):
if listname == mlist.internal_name():
xlist = mlist
else:
......
......@@ -26,6 +26,9 @@
import os
from munepy import Enum
def seconds(s): return s
def minutes(m): return m * 60
def hours(h): return h * 60 * 60
......@@ -79,7 +82,7 @@ SITE_OWNER_ADDRESS = '[email protected]'
DEFAULT_HOST_NAME = None
DEFAULT_URL = None
HOME_PAGE = 'index.html'
HOME_PAGE = 'index.html'
# Normally when a site administrator authenticates to a web page with the site
# password, they get a cookie which authorizes them as the list admin. It
......@@ -104,6 +107,11 @@ PASSWORD_SCHEME = 'ssha'
# Database options
#####
# Initialization function for creating the IListManager, IUserManager, and
# IMessageManager objects, as a Python dotted name. This function must take
# zero arguments.
MANAGERS_INIT_FUNCTION = 'Mailman.database.initialize'
# Use this to set the SQLAlchemy database engine URL. You generally have one
# primary database connection for all of Mailman. List data and most rosters
# will store their data in this database, although external rosters may access
......@@ -467,6 +475,15 @@ NNTP_REWRITE_DUPLICATE_HEADERS = [
('mime-version', 'X-MIME-Version'),
]
# Some list posts and mail to the -owner address may contain DomainKey or
# DomainKeys Identified Mail (DKIM) signature headers <http://www.dkim.org/>.
# Various list transformations to the message such as adding a list header or
# footer or scrubbing attachments or even reply-to munging can break these
# signatures. It is generally felt that these signatures have value, even if
# broken and even if the outgoing message is resigned. However, some sites
# may wish to remove these headers by setting this to Yes.
REMOVE_DKIM_HEADERS = No
# All `normal' messages which are delivered to the entire list membership go
# through this pipeline of handler modules. Lists themselves can override the
# global pipeline by defining a `pipeline' attribute.
......@@ -525,6 +542,8 @@ OWNER_PIPELINE = [
# - propagate -- Boolean specifying whether to propagate log message from this
# logger to the root "mailman" logger. You cannot override
# settings for the root logger.
#
# The file name may be absolute, or relative to Mailman's etc directory.
LOG_CONFIG_FILE = None
# This defines log format strings for the SMTPDirect delivery module (see
......@@ -664,9 +683,9 @@ VERP_DELIVERY_INTERVAL = 0
# friendly Subject: on the message, but requires cooperation from the MTA.
# Format is like VERP_FORMAT above, but with the following substitutions:
#
# %(addr)s -- the list-confirm mailbox will be set here
# %(cookie)s -- the confirmation cookie will be set here
VERP_CONFIRM_FORMAT = '%(addr)s+%(cookie)s'
# $address -- the list-confirm address
# $cookie -- the confirmation cookie
VERP_CONFIRM_FORMAT = '$address+$cookie'
# This is analogous to VERP_REGEXP, but for splitting apart the
# VERP_CONFIRM_FORMAT. MUAs have been observed that mung
......@@ -1295,6 +1314,24 @@ ReceiveNonmatchingTopics = 64
Moderate = 128
DontReceiveDuplicates = 256
class DeliveryMode(Enum):
# Non-digest delivery
Regular = 1
# Digest delivery modes
MIME = 2
Plain = 3
class DeliveryStatus(Enum):
Enabled = 0
# Disabled reason
Unknown = 1
ByUser = 2
ByAdmin = 3
ByBounce = 4
# A mapping between short option tags and their flag
OPTINFO = {'hide' : ConcealSubscription,
'nomail' : DisableDelivery,
......
......@@ -212,3 +212,33 @@ class BadPasswordSchemeError(PasswordError):
def __str__(self):
return 'A bad password scheme was given: %s' % self.scheme_name
class UserError(MailmanError):
"""A general user-related error occurred."""
class RosterError(UserError):
"""A roster-related error occurred."""
class RosterExistsError(RosterError):
"""The named roster already exists."""
class AddressError(MailmanError):
"""A general address-related error occurred."""
class ExistingAddressError(AddressError):
"""The given email address already exists."""
class AddressAlreadyLinkedError(AddressError):
"""The address is already linked to a user."""
class AddressNotLinkedError(AddressError):
"""The address is not linked to the user."""
......@@ -23,7 +23,7 @@ from Mailman import Utils
from Mailman import i18n
from Mailman import mm_cfg
from Mailman.Gui.GUIBase import GUIBase
from Mailman.database.languages import Language
from Mailman.database.tables.languages import Language
_ = i18n._
......
......@@ -25,9 +25,12 @@ and it will also give the MTA the opportunity to regenerate valid keys
originating at the Mailman server for the outgoing message.
"""
from Mailman.configuration import config
def process(mlist, msg, msgdata):
del msg['domainkey-signature']
del msg['dkim-signature']
del msg['authentication-results']
if config.REMOVE_DKIM_HEADERS:
del msg['domainkey-signature']
del msg['dkim-signature']
del msg['authentication-results']
......@@ -174,7 +174,19 @@ def process(mlist, msg, msgdata=None):
if ctype == 'text/plain':
# We need to choose a charset for the scrubbed message, so we'll
# arbitrarily pick the charset of the first text/plain part in the
# message. Also get the RFC 3676 stuff from this part.
# message.
#
# Also get the RFC 3676 stuff from this part. This seems to
# work okay for scrub_nondigest. It will also work as far as
# scrubbing messages for the archive is concerned, but Pipermail
# doesn't pay any attention to the RFC 3676 parameters. The plain
# format digest is going to be a disaster in any case as some of
# messages will be format="flowed" and some not. ToDigest creates
# its own Content-Type: header for the plain digest which won't
# have RFC 3676 parameters. If the message Content-Type: headers
# are retained for display in the digest, the parameters will be
# there for information, but not for the MUA. This is the best we
# can do without having get_payload() process the parameters.
if charset is None:
charset = part.get_content_charset(lcset)
format = part.get_param('format')
......@@ -318,7 +330,8 @@ URL: %(url)s
partcharset = part.get_content_charset('us-ascii')
try:
t = unicode(t, partcharset, 'replace')
except (UnicodeError, LookupError, ValueError, TypeError):
except (UnicodeError, LookupError, ValueError, TypeError,
AssertionError):
# What is the cause to come this exception now ?
# Replace funny characters. We use errors='replace'.
u = unicode(t, 'ascii', 'replace')
......@@ -331,6 +344,13 @@ URL: %(url)s
charsets.append(partcharset)
# Now join the text and set the payload
sep = _('-------------- next part --------------\n')
# The i18n separator is in the list's charset. Coerce it to the
# message charset.
try:
s = unicode(sep, lcset, 'replace')
sep = s.encode(charset, 'replace')
except (UnicodeError, LookupError, ValueError):
pass
rept = sep.join(text)
# Replace entire message with text and scrubbed notice.
# Try with message charsets and utf-8
......
This diff is collapsed.
......@@ -321,6 +321,16 @@ class LockFile:
if self._owned:
self.finalize()
# Python 2.5 context manager protocol support.
def __enter__(self):
self.lock()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.unlock()
# Don't suppress any exception that might have occurred.
return False
# Use these only if you're transfering ownership to a child process across
# a fork. Use at your own risk, but it should be race-condition safe.
# _transfer_to() is called in the parent, passing in the pid of the child.
......
This diff is collapsed.
......@@ -44,7 +44,7 @@ SHELL= /bin/sh
MODULES= $(srcdir)/*.py
SUBDIRS= Cgi Archiver Handlers Bouncers Queue MTA Gui Commands \
bin database testing
bin database docs ext interfaces testing
# Modes for directories and executables created by the install
# process. Default to group-writable directories but
......
......@@ -17,6 +17,8 @@
"""Track pending actions which require confirmation."""
from __future__ import with_statement
import os
import sha
import time
......@@ -51,7 +53,7 @@ _default = object()
class Pending:
def InitTempVars(self):
self.__pendfile = os.path.join(self.fullpath(), 'pending.pck')
self._pendfile = os.path.join(self.full_path, 'pending.pck')
def pend_new(self, op, *content, **kws):
"""Create a new entry in the pending database, returning cookie for it.
......@@ -87,14 +89,12 @@ class Pending:
def __load(self):
try:
fp = open(self.__pendfile)
with open(self._pendfile) as fp:
return cPickle.load(fp)
except IOError, e:
if e.errno <> errno.ENOENT: raise
if e.errno <> errno.ENOENT:
raise
return {'evictions': {}}
try:
return cPickle.load(fp)
finally:
fp.close()
def __save(self, db):
evictions = db['evictions']
......@@ -112,15 +112,12 @@ class Pending:
if not db.has_key(cookie):
del evictions[cookie]
db['version'] = config.PENDING_FILE_SCHEMA_VERSION
tmpfile = '%s.tmp.%d.%d' % (self.__pendfile, os.getpid(), now)
fp = open(tmpfile, 'w')
try:
tmpfile = '%s.tmp.%d.%d' % (self._pendfile, os.getpid(), now)
with open(tmpfile, 'w') as fp:
cPickle.dump(db, fp)
fp.flush()
os.fsync(fp.fileno())
finally:
fp.close()
os.rename(tmpfile, self.__pendfile)
os.rename(tmpfile, self._pendfile)
def pend_confirm(self, cookie, expunge=True):
"""Return data for cookie, or None if not found.
......
......@@ -48,7 +48,6 @@ import asyncore
from email.utils import parseaddr
from Mailman import Utils
from Mailman.Message import Message
from Mailman.Queue.Runner import Runner
from Mailman.Queue.sbcache import get_switchboard
......@@ -122,7 +121,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
# since the set of mailing lists could have changed. However, on
# a big site this could be fairly expensive, so we may need to
# cache this in some way.
listnames = Utils.list_names()
listnames = set(config.list_manager.names)
# Parse the message data. XXX Should we reject the message
# immediately if it has defects? Usually only spam has defects.
msg = email.message_from_string(data, Message)
......
......@@ -56,7 +56,6 @@ import logging
from email.Parser import Parser
from email.Utils import parseaddr
from Mailman import Utils
from Mailman.Message import Message
from Mailman.Queue.Runner import Runner
from Mailman.Queue.sbcache import get_switchboard
......@@ -101,9 +100,8 @@ class MaildirRunner(Runner):
self._parser = Parser(Message)
def _oneloop(self):
# Refresh this each time through the list. BAW: could be too
# expensive.
listnames = Utils.list_names()
# Refresh this each time through the list.
listnames = list(config.list_manager.names)
# Cruise through all the files currently in the new/ directory
try:
files = os.listdir(self._dir)
......
......@@ -92,16 +92,16 @@ class Runner:
# Ask the switchboard for the message and metadata objects
# associated with this filebase.
msg, msgdata = self._switchboard.dequeue(filebase)
except email.Errors.MessageParseError, e:
# It's possible to get here if the message was stored in the
# pickle in plain text, and the metadata had a _parsemsg key
# that was true, /and/ if the message had some bogosity in
# it. It's almost always going to be spam or bounced spam.
# There's not much we can do (and we didn't even get the
# metadata, so just log the exception and continue.
except Exception, e:
# This used to just catch email.Errors.MessageParseError,
# but other problems can occur in message parsing, e.g.
# ValueError, and exceptions can occur in unpickling too.
# We don't want the runner to die, so we just log and skip
# this entry, but preserve it for analysis.
self._log(e)
log.error('Ignoring unparseable message: %s', filebase)
self._switchboard.finish(filebase)
log.error('Skipping and preserving unparseable message: %s',
filebase)
self._switchboard.finish(filebase, preserve=True)
continue
try:
self._onefile(msg, msgdata)
......@@ -116,9 +116,21 @@ class Runner:
self._log(e)
# Put a marker in the metadata for unshunting
msgdata['whichq'] = self._switchboard.whichq()
new_filebase = self._shunt.enqueue(msg, msgdata)
log.error('SHUNTING: %s', new_filebase)
self._switchboard.finish(filebase)
# It is possible that shunting can throw an exception, e.g. a
# permissions problem or a MemoryError due to a really large
# message. Try to be graceful.
try:
new_filebase = self._shunt.enqueue(msg, msgdata)
log.error('SHUNTING: %s', new_filebase)
self._switchboard.finish(filebase)
except Exception, e:
# The message wasn't successfully shunted. Log the
# exception and try to preserve the original queue entry
# for possible analysis.
self._log(e)
log.error('SHUNTING FAILED, preserving original entry: %s',
filebase)
self._switchboard.finish(filebase, preserve=True)
# Other work we want to do each time through the loop
Utils.reap(self._kids, once=True)
self._doperiodic()
......
......@@ -149,12 +149,20 @@ class Switchboard:
msg = email.message_from_string(msg, Message.Message)
return msg, data
def finish(self, filebase):
def finish(self, filebase, preserve=False):
bakfile = os.path.join(self.__whichq, filebase + '.bak')
try:
os.unlink(bakfile)
if preserve:
psvfile = os.path.join(config.SHUNTQUEUE_DIR,
filebase + '.psv')
# Create the directory if it doesn't yet exist.
Utils.makedirs(config.SHUNTQUEUE_DIR, 0770)
os.rename(bakfile, psvfile)
else:
os.unlink(bakfile)
except EnvironmentError, e:
elog.exception('Failed to unlink backup file: %s', bakfile)
elog.exception('Failed to unlink/preserve backup file: %s',
bakfile)
def files(self, extension='.pck'):
times = {}
......
......@@ -40,7 +40,6 @@ from email.Errors import HeaderParseError
from string import ascii_letters, digits, whitespace
from Mailman import Errors
from Mailman import database
from Mailman import passwords
from Mailman.SafeDict import SafeDict
from Mailman.configuration import config
......@@ -66,13 +65,13 @@ log = logging.getLogger('mailman.error')
def list_exists(fqdn_listname):
"""Return true iff list `fqdn_listname' exists."""
listname, hostname = split_listname(fqdn_listname)
return bool(database.find_list(listname, hostname))
return bool(config.list_manager.find_list(listname, hostname))
def list_names():
"""Return the fqdn names of all lists in default list directory."""
return ['%[email protected]%s' % (listname, hostname)
for listname, hostname in database.get_list_names()]
for listname, hostname in config.list_manager.get_list_names()]
def split_listname(listname):
......@@ -81,8 +80,10 @@ def split_listname(listname):
return listname, config.DEFAULT_EMAIL_HOST
def fqdn_listname(listname):
return AT.join(split_listname(listname))
def fqdn_listname(listname, hostname=None):
if hostname is None:
return AT.join(split_listname(listname))
return AT.join((listname, hostname))
......
......@@ -20,7 +20,6 @@ import optparse
from Mailman import Errors
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.configuration import config
from Mailman.i18n import _
......@@ -52,7 +51,7 @@ def main():
opts, args, parser = parseargs()
config.load(opts.config)
listnames = set(args or Utils.list_names())
listnames = set(args or config.list_manager.names)
if not listnames:
print _('Nothing to do.')
sys.exit(0)
......
......@@ -114,13 +114,10 @@ def main():
# Cull duplicates
domains = set(opts.domains)
if opts.all:
listnames = set(Utils.list_names())
else:
listnames = set(opts.listnames)
listnames = set(config.list_manager.names if opts.all else opts.listnames)
if domains:
for name in Utils.list_names():
for name in config.list_manager.names:
mlist = openlist(name)
if mlist.host_name in domains:
listnames.add(name)
......
......@@ -132,7 +132,7 @@ def main():
i18n.set_language(config.DEFAULT_SERVER_LANGUAGE)
for name in Utils.list_names():
for name in config.list_manager.names:
# The list must be locked in order to open the requests database
mlist = MailList.MailList(name)
try:
......
......@@ -23,7 +23,6 @@ from Mailman import Errors
from Mailman import MailList
from Mailman import MemberAdaptor
from Mailman import Pending
from Mailman import Utils
from Mailman import Version
from Mailman import loginit
from Mailman.Bouncer import _BounceInfo
......@@ -122,7 +121,7 @@ def main():
elog = logging.getLogger('mailman.error')
blog = logging.getLogger('mailman.bounce')
listnames = set(opts.listnames or Utils.list_names())
listnames = set(opts.listnames or config.list_manager.names)
who = tuple(opts.who)
msg = _('[disabled by periodic sweep and cull, no message available]')
......
......@@ -31,7 +31,6 @@ from xml.sax.saxutils import escape
from Mailman import Defaults
from Mailman import Errors
from Mailman import MemberAdaptor
from Mailman import Utils
from Mailman import Version
from Mailman.MailList import MailList
from Mailman.configuration import config
......@@ -308,7 +307,7 @@ def main():
listname = '%[email protected]%s' % (listname, config.DEFAULT_EMAIL_HOST)
listnames.append(listname)
else:
listnames = Utils.list_names()
listnames = config.list_manager.names
dumper.dump(listnames)
dumper.close()
finally:
......
......@@ -21,7 +21,6 @@ import optparse
from Mailman import Errors
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.configuration import config
from Mailman.i18n import _
......@@ -79,9 +78,8 @@ def main():
parser, opts, args = parseargs()
config.load(opts.config)
if not opts.listnames:
opts.listnames = Utils.list_names()
includes = set(listname.lower() for listname in opts.listnames)
listnames = opts.listnames or config.list_manager.names
includes = set(listname.lower() for listname in listnames)
excludes = set(listname.lower() for listname in opts.excludes)
listnames = includes - excludes
......
......@@ -164,7 +164,7 @@ def poll_newsgroup(mlist, conn, first, last, glock):
def process_lists(glock):
for listname in Utils.list_names():
for listname in config.list_manager.names:
glock.refresh()
# Open the list unlocked just to check to see if it is gating news to
# mail. If not, we're done with the list. Otherwise, lock the list
......
......@@ -21,7 +21,6 @@ import sys
import optparse
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.configuration import config
from Mailman.i18n import _
......@@ -68,7 +67,7 @@ def main():
lock.lock()
# Group lists by virtual hostname
mlists = {}
for listname in Utils.list_names():
for listname in config.list_manager.names:
mlist = MailList.MailList(listname, lock=False)
mlists.setdefault(mlist.host_name, []).append(mlist)
try:
......
......@@ -19,7 +19,6 @@ import optparse
from Mailman import Defaults
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.i18n import _
from Mailman.initialize import initialize
......@@ -66,12 +65,10 @@ def main():
parser, opts, args = parseargs()
initialize(opts.config)
names = list(Utils.list_names())
names.sort()
mlists = []
longest = 0
for n in names:
for n in sorted(config.list_manager.names):
mlist = MailList.MailList(n, lock=False)
if opts.advertised and not mlist.advertised:
continue
......
......@@ -18,7 +18,6 @@
import sys
import optparse
from Mailman import Utils
from Mailman import Version
from Mailman.MailList import MailList
from Mailman.configuration import config
......@@ -55,7 +54,7 @@ def main():
parser, opts, args = parseargs()
config.load(opts.config)
listnames = args or Utils.list_names()
listnames = set(args or config.list_manager.names)
bylist = {}
for listname in listnames:
......
......@@ -25,7 +25,6 @@ except ImportError:
sys.exit(0)
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.configuration import config
from Mailman.i18n import _
......@@ -86,7 +85,7 @@ def main():
return
# Process all the specified lists
for listname in set(args or Utils.list_names()):
for listname in set(args or config.list_manager.names):
mlist = MailList.MailList(listname, lock=False)
if not mlist.archive:
continue
......
......@@ -19,7 +19,6 @@ import sys
import optparse
from Mailman import MailList
from Mailman import Utils
from Mailman import Version
from Mailman.i18n import _
from Mailman.initialize import initialize
......@@ -59,7 +58,7 @@ def main():
opts, args, parser = parseargs()
initialize(opts.config)
for listname in set(opts.listnames or Utils.list_names()):