Commit de8c204f authored by Barry Warsaw's avatar Barry Warsaw

Add a new plugin architecture.

This allows third parties to add initialization hooks, REST endpoints, 
and additional components.  Given by Jan Jancar.
parents f847e154 ae0042a9
......@@ -60,6 +60,7 @@ Table of Contents
src/mailman/docs/mta
src/mailman/docs/postorius
src/mailman/docs/hyperkitty
src/mailman/plugins/docs/intro
src/mailman/docs/contribute
src/mailman/docs/STYLEGUIDE
src/mailman/docs/internationalization
......
......@@ -26,4 +26,4 @@ from public import public
@public
def initialize():
"""Initialize the email commands."""
add_components('mailman.commands', IEmailCommand, config.commands)
add_components('commands', IEmailCommand, config.commands)
=====
Hooks
=====
Mailman defines two initialization hooks, one which is run early in the
initialization process and the other run late in the initialization process.
Hooks name an importable callable so it must be accessible on ``sys.path``.
::
>>> import os, sys
>>> from mailman.config import config
>>> config_directory = os.path.dirname(config.filename)
>>> sys.path.insert(0, config_directory)
>>> hook_path = os.path.join(config_directory, 'hooks.py')
>>> with open(hook_path, 'w') as fp:
... print("""\
... counter = 1
... def pre_hook():
... global counter
... print('pre-hook:', counter)
... counter += 1
...
... def post_hook():
... global counter
... print('post-hook:', counter)
... counter += 1
... """, file=fp)
>>> fp.close()
Pre-hook
========
We can set the pre-hook in the configuration file.
>>> config_path = os.path.join(config_directory, 'hooks.cfg')
>>> with open(config_path, 'w') as fp:
... print("""\
... [meta]
... extends: test.cfg
...
... [mailman]
... pre_hook: hooks.pre_hook
... """, file=fp)
The hooks are run in the second and third steps of initialization. However,
we can't run those initialization steps in process, so call a command line
script that will produce no output to force the hooks to run.
::
>>> import subprocess
>>> from mailman.testing.layers import ConfigLayer
>>> def call():
... exe = os.path.join(os.path.dirname(sys.executable), 'mailman')
... env = os.environ.copy()
... env.update(
... MAILMAN_CONFIG_FILE=config_path,
... PYTHONPATH=config_directory,
... )
... test_cfg = os.environ.get('MAILMAN_EXTRA_TESTING_CFG')
... if test_cfg is not None:
... env['MAILMAN_EXTRA_TESTING_CFG'] = test_cfg
... proc = subprocess.Popen(
... [exe, 'lists', '--domain', 'ignore', '-q'],
... cwd=ConfigLayer.root_directory, env=env,
... universal_newlines=True,
... stdout=subprocess.PIPE, stderr=subprocess.PIPE)
... stdout, stderr = proc.communicate()
... assert proc.returncode == 0, stderr
... print(stdout)
>>> call()
pre-hook: 1
<BLANKLINE>
>>> os.remove(config_path)
Post-hook
=========
We can set the post-hook in the configuration file.
::
>>> with open(config_path, 'w') as fp:
... print("""\
... [meta]
... extends: test.cfg
...
... [mailman]
... post_hook: hooks.post_hook
... """, file=fp)
>>> call()
post-hook: 1
<BLANKLINE>
>>> os.remove(config_path)
Running both hooks
==================
We can set the pre- and post-hooks in the configuration file.
::
>>> with open(config_path, 'w') as fp:
... print("""\
... [meta]
... extends: test.cfg
...
... [mailman]
... pre_hook: hooks.pre_hook
... post_hook: hooks.post_hook
... """, file=fp)
>>> call()
pre-hook: 1
post-hook: 2
<BLANKLINE>
......@@ -16,7 +16,6 @@
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""The 'mailman' command dispatcher."""
import click
from contextlib import ExitStack
......@@ -35,17 +34,21 @@ class Subcommands(click.MultiCommand):
def __init__(self, *args, **kws):
super().__init__(*args, **kws)
self._commands = {}
# Look at all modules in the mailman.bin package and if they are
# prepared to add a subcommand, let them do so. I'm still undecided as
# to whether this should be pluggable or not. If so, then we'll
# probably have to partially parse the arguments now, then initialize
# the system, then find the plugins. Punt on this for now.
add_components('mailman.commands', ICLISubCommand, self._commands)
self._loaded = False
def _load(self):
# Load commands lazily as commands in plugins can only be found after
# the configuration file is loaded.
if not self._loaded:
add_components('commands', ICLISubCommand, self._commands)
self._loaded = True
def list_commands(self, ctx):
return sorted(self._commands) # pragma: nocover
def list_commands(self, ctx): # pragma: nocover
self._load()
return sorted(self._commands)
def get_command(self, ctx, name):
self._load()
try:
return self._commands[name].command
except KeyError as error:
......@@ -85,10 +88,11 @@ class Subcommands(click.MultiCommand):
super().format_commands(ctx, formatter)
@click.group(
cls=Subcommands,
context_settings=dict(help_option_names=['-h', '--help']))
@click.pass_context
def initialize_config(ctx, param, value):
if not ctx.resilient_parsing:
initialize(value)
@click.option(
'-C', '--config', 'config_file',
envvar='MAILMAN_CONFIG_FILE',
......@@ -96,7 +100,12 @@ class Subcommands(click.MultiCommand):
help=_("""\
Configuration file to use. If not given, the environment variable
MAILMAN_CONFIG_FILE is consulted and used if set. If neither are given, a
default configuration file is loaded."""))
default configuration file is loaded."""),
is_eager=True, callback=initialize_config)
@click.group(
cls=Subcommands,
context_settings=dict(help_option_names=['-h', '--help']))
@click.pass_context
@click.version_option(MAILMAN_VERSION_FULL, message='%(version)s')
@public
def main(ctx, config_file):
......@@ -106,6 +115,4 @@ def main(ctx, config_file):
Copyright 1998-2017 by the Free Software Foundation, Inc.
http://www.list.org
"""
# Initialize the system. Honor the -C flag if given.
initialize(config_file)
# click handles dispatching to the subcommand via the Subcommands class.
......@@ -45,22 +45,23 @@ SUBPROC_START_WAIT = timedelta(seconds=20)
# Environment variables to forward into subprocesses.
PRESERVE_ENVS = (
'COVERAGE_PROCESS_START',
'MAILMAN_EXTRA_TESTING_CFG',
'LANG',
'LANGUAGE',
'LC_CTYPE',
'LC_NUMERIC',
'LC_TIME',
'LC_ADDRESS',
'LC_ALL',
'LC_COLLATE',
'LC_MONETARY',
'LC_CTYPE',
'LC_IDENTIFICATION',
'LC_MEASUREMENT',
'LC_MESSAGES',
'LC_PAPER',
'LC_MONETARY',
'LC_NAME',
'LC_ADDRESS',
'LC_NUMERIC',
'LC_PAPER',
'LC_TELEPHONE',
'LC_MEASUREMENT',
'LC_IDENTIFICATION',
'LC_ALL',
'LC_TIME',
'MAILMAN_EXTRA_TESTING_CFG',
'PYTHONPATH',
)
......@@ -188,6 +189,9 @@ class PIDWatcher:
def __init__(self):
self._pids = {}
def __contains__(self, pid):
return pid in self._pids.keys()
def __iter__(self):
# Safely iterate over all the keys in the dictionary. Because
# asynchronous signals are involved, the dictionary's size could
......@@ -312,16 +316,16 @@ class Loop:
env = {'MAILMAN_UNDER_MASTER_CONTROL': '1'}
# Craft the command line arguments for the exec() call.
rswitch = '--runner=' + spec
# Wherever master lives, so too must live the runner script.
exe = os.path.join(config.BIN_DIR, 'runner')
# config.PYTHON, which is the absolute path to the Python interpreter,
# must be given as argv[0] due to Python's library search algorithm.
args = [sys.executable, sys.executable, exe, rswitch]
# Always pass the explicit path to the configuration file to the
# sub-runners. This avoids any debate about which cfg file is used.
config_file = (config.filename if self._config_file is None
else self._config_file)
args.extend(['-C', config_file])
# Wherever master lives, so too must live the runner script.
exe = os.path.join(config.BIN_DIR, 'runner') # pragma: nocover
# config.PYTHON, which is the absolute path to the Python interpreter,
# must be given as argv[0] due to Python's library search algorithm.
args = [sys.executable, sys.executable, exe, # pragma: nocover
'-C', config_file, rswitch]
log = logging.getLogger('mailman.runner')
log.debug('starting: %s', args)
# We must pass this environment variable through if it's set,
......@@ -399,9 +403,13 @@ class Loop:
except ChildProcessError:
# No children? We're done.
break
except InterruptedError: # pragma: nocover
except InterruptedError: # pragma: nocover
# If the system call got interrupted, just restart it.
continue
if pid not in self._kids: # pragma: nocover
# This is not a runner subprocess that we own. E.g. maybe a
# plugin started it.
continue
# Find out why the subprocess exited by getting the signal
# received or exit status.
if os.WIFSIGNALED(status):
......
......@@ -151,14 +151,12 @@ def main(ctx, config_file, verbose, list_runners, once, runner_spec):
run this way, the environment variable $MAILMAN_UNDER_MASTER_CONTROL
will be set which subtly changes some error handling behavior.
"""
global log
if runner_spec is None and not list_runners:
ctx.fail(_('No runner name given.'))
# Initialize the system. Honor the -C flag if given.
initialize(config_file, verbose)
log = logging.getLogger('mailman.runner')
if verbose:
......
......@@ -29,6 +29,7 @@ from mailman.interfaces.command import ICLISubCommand
from mailman.testing.layers import ConfigLayer
from mailman.utilities.datetime import now
from mailman.utilities.modules import add_components
from pkg_resources import resource_filename
from unittest.mock import patch
......@@ -38,7 +39,19 @@ class TestMailmanCommand(unittest.TestCase):
def setUp(self):
self._command = CliRunner()
def test_mailman_command_without_subcommand_prints_help(self):
def test_mailman_command_config(self):
config_path = resource_filename('mailman.testing', 'testing.cfg')
with patch('mailman.bin.mailman.initialize') as init:
self._command.invoke(main, ('-C', config_path, 'info'))
init.assert_called_once_with(config_path)
def test_mailman_command_no_config(self):
with patch('mailman.bin.mailman.initialize') as init:
self._command.invoke(main, ('info',))
init.assert_called_once_with(None)
@patch('mailman.bin.mailman.initialize')
def test_mailman_command_without_subcommand_prints_help(self, mock):
# Issue #137: Running `mailman` without a subcommand raises an
# AttributeError.
result = self._command.invoke(main)
......@@ -49,14 +62,15 @@ class TestMailmanCommand(unittest.TestCase):
self.assertEqual(lines[0], 'Usage: main [OPTIONS] COMMAND [ARGS]...')
# The help output includes a list of subcommands, in sorted order.
commands = {}
add_components('mailman.commands', ICLISubCommand, commands)
add_components('commands', ICLISubCommand, commands)
help_commands = list(
line.split()[0].strip()
for line in lines[-len(commands):]
)
self.assertEqual(sorted(commands), help_commands)
def test_mailman_command_with_bad_subcommand_prints_help(self):
@patch('mailman.bin.mailman.initialize')
def test_mailman_command_with_bad_subcommand_prints_help(self, mock):
# Issue #137: Running `mailman` without a subcommand raises an
# AttributeError.
result = self._command.invoke(main, ('not-a-subcommand',))
......
......@@ -178,9 +178,12 @@ def listaddr(mlist):
def requestaddr(mlist):
print(mlist.request_address)
All run methods take at least one argument, the mailing list object to operate
Run methods take at least one argument, the mailing list object to operate
on. Any additional arguments given on the command line are passed as
positional arguments to the callable."""))
positional arguments to the callable.
If -l is not given then you can run a function that takes no arguments.
"""))
print()
print(_("""\
You can print the list's posting address by running the following from the
......@@ -232,11 +235,16 @@ Mailman will do this for you (assuming no errors occured)."""))
@click.option(
'--run', '-r',
help=_("""\
Run a script on a mailing list. The argument is the module path to a
callable. This callable will be imported and then called with the mailing
list as the first argument. If additional arguments are given at the end
of the command line, they are passed as subsequent positional arguments to
the callable. For additional help, see --details.
Run a script. The argument is the module path to a callable. This
callable will be imported and then, if --listspec/-l is also given, is
called with the mailing list as the first argument. If additional
arguments are given at the end of the command line, they are passed as
subsequent positional arguments to the callable. For additional help, see
--details.
If no --listspec/-l argument is given, the script function being called is
called with no arguments.
"""))
@click.option(
'--details',
......@@ -270,9 +278,8 @@ def shell(ctx, interactive, run, listspec, run_args):
# without the dot is allowed.
dotted_name = (run if '.' in run else '{0}.{0}'.format(run))
if listspec is None:
ctx.fail(_('--run requires a mailing list'))
# Parse the run arguments so we can pass them into the run method.
if listspec.startswith('^'):
r = call_name(dotted_name, *run_args)
elif listspec.startswith('^'):
r = {}
cre = re.compile(listspec, re.IGNORECASE)
for mlist in list_manager.mailing_lists:
......
......@@ -45,6 +45,7 @@ key, along with the names of the corresponding sections.
[logging.http] path: mailman.log
[logging.locks] path: mailman.log
[logging.mischief] path: mailman.log
[logging.plugins] path: plugins.log
[logging.root] path: mailman.log
[logging.runner] path: mailman.log
[logging.smtp] path: smtp.log
......
......@@ -59,7 +59,6 @@ definition.
CFG_FILE = .../test.cfg
DATA_DIR = /var/lib/mailman/data
ETC_DIR = /etc
EXT_DIR = /etc/mailman.d
LIST_DATA_DIR = /var/lib/mailman/lists
LOCK_DIR = /var/lock/mailman
LOCK_FILE = /var/lock/mailman/master.lck
......
......@@ -29,6 +29,7 @@ from mailman.interfaces.usermanager import IUserManager
from mailman.testing.helpers import configuration
from mailman.testing.layers import ConfigLayer
from mailman.utilities.modules import hacked_sys_modules
from types import ModuleType
from unittest.mock import MagicMock, patch
try:
......@@ -40,6 +41,7 @@ except ImportError:
class TestShell(unittest.TestCase):
layer = ConfigLayer
maxDiff = None
def setUp(self):
self._command = CliRunner()
......@@ -156,12 +158,12 @@ class TestShell(unittest.TestCase):
'Error: No such list: ant.example.com\n')
def test_run_without_listspec(self):
results = self._command.invoke(shell, ('--run', 'something'))
self.assertEqual(results.exit_code, 2)
self.assertEqual(
results.output,
'Usage: shell [OPTIONS] [RUN_ARGS]...\n\n'
'Error: --run requires a mailing list\n')
something = ModuleType('something')
something.something = lambda: print('I am a something!')
with hacked_sys_modules('something', something):
results = self._command.invoke(shell, ('--run', 'something'))
self.assertEqual(results.exit_code, 0)
self.assertEqual(results.output, 'I am a something!\n')
def test_run_bogus_listspec(self):
results = self._command.invoke(
......
......@@ -40,6 +40,19 @@ from zope.interface import implementer
SPACE = ' '
SPACERS = '\n'
DIR_NAMES = (
'archive',
'bin',
'cache',
'data',
'etc',
'list_data',
'lock',
'log',
'messages',
'queue',
)
MAILMAN_CFG_TEMPLATE = """\
# AUTOMATICALLY GENERATED BY MAILMAN ON {} UTC
......@@ -76,6 +89,7 @@ class Configuration:
self.handlers = {}
self.pipelines = {}
self.commands = {}
self.plugins = {}
self.password_context = None
self.db = None
......@@ -158,8 +172,7 @@ class Configuration:
else category.template_dir),
)
# Directories.
for name in ('archive', 'bin', 'cache', 'data', 'etc', 'ext',
'list_data', 'lock', 'log', 'messages', 'queue'):
for name in DIR_NAMES:
key = '{}_dir'.format(name)
substitutions[key] = getattr(category, key)
# Files.
......@@ -247,6 +260,35 @@ class Configuration:
archiver.is_enabled = as_boolean(section.enable)
yield archiver
@property
def plugin_configs(self):
"""Return all the plugin configuration sections."""
plugin_sections = self._config.getByCategory('plugin', [])
for section in plugin_sections:
# 2017-08-27 barry: There's a fundamental constraint imposed by
# lazr.config, namely that we have to use a .master section instead
# of a .template section in the schema.cfg, or user supplied
# configuration files cannot define new [plugin.*] sections. See
# https://bugs.launchpad.net/lazr.config/+bug/310619 for
# additional details.
#
# However, this means that [plugin.master] will show up in the
# categories retrieved above. But 'master' is not a real plugin,
# so we need to skip it (e.g. otherwise we'll get log warnings
# about plugin.master being disabled, etc.). This imposes an
# additional limitation though in that users cannot define a
# plugin named 'master' because you can't override a master
# section with a real section. There's no good way around this so
# we just have to live with this limitation.
if section.name == 'plugin.master':
continue
# The section.name will be something like 'plugin.example', but we
# only want the 'example' part as the name of the plugin. We
# could split on dots, but lazr.config gives us a different way.
# `category_and_section_names` is a 2-tuple of e.g.
# ('plugin', 'example'), so just grab the last element.
yield section.category_and_section_names[1], section
@property
def language_configs(self):
"""Iterate over all the language configuration sections."""
......
......@@ -54,14 +54,6 @@ pending_request_life: 3d
# How long should files be saved before they are evicted from the cache?
cache_life: 7d
# A callable to run with no arguments early in the initialization process.
# This runs before database initialization.
pre_hook:
# A callable to run with no arguments late in the initialization process.
# This runs after adapters are initialized.
post_hook:
# Which paths.* file system layout to use.
layout: here
......@@ -82,6 +74,51 @@ html_to_plain_text_command: /usr/bin/lynx -dump $filename
# unpredictable.
listname_chars: [-_.0-9a-z]
# These hooks are deprecated, but are kept here so as not to break existing
# configuration files. However, these hooks are not run. Define a plugin
# instead.
pre_hook:
post_hook:
# Plugin configuration section template.
#
# To add a plugin, instantiate this section (changing `master` to whatever
# your plugin's name is), and define at least a `path` and a `class`. When
# the plugin is loaded, its subpackages will be search for components matching
# the following interfaces:
#
# - IChain for new chains
# - ICliSubCommand - `mailman` subcommands
# - IEmailCommand - new email commands
# - IHandler for new handlers
# - IPipeline for new pipelines
# - IRule for new rules
# - IStyle for new styles.
#
# See the IPlugin interface for more details.
[plugin.master]
# The full Python import path for you IPlugin implementing class. It is
# required to provide this.
class:
# Whether to enable this plugin or not.
enabled: no
# Additional configuration file for this plugin. If the value starts with
# `python:` it is a Python import path, in which case the value should not
# include the trailing .cfg (although the file is required to have this
# suffix). Without `python:`, it is a file system path, and must be an
# absolute path, since no guarantees are made about the current working
# directory.
configuration:
# Package (as a dotted Python import path) to search for components that this
# plugin wants to add, such as ISTyles, IRules, etc. If not given, the
# plugin's name is used.
component_package:
[shell]
# `mailman shell` (also `withlist`) gives you an interactive prompt that you
......@@ -138,8 +175,6 @@ data_dir: $var_dir/data
cache_dir: $var_dir/cache
# Directory for configuration files and such.
etc_dir: $var_dir/etc
# Directory containing Mailman plugins.
ext_dir: $var_dir/ext
# Directory where the default IMessageStore puts its messages.
messages_dir: $var_dir/messages
# Directory for archive backends to store their messages in. Archivers should
......@@ -261,6 +296,7 @@ debug: no
# - http -- Internal wsgi-based web interface
# - locks -- Lock state changes
# - mischief -- Various types of hostile activity