Commit ece55099 authored by Aurélien Bompard's avatar Aurélien Bompard Committed by Barry Warsaw

Convert header_filter_rules from 2.1 to header_matches

parent 6a229576
......@@ -24,9 +24,11 @@ __all__ = [
import os
import re
import sys
import codecs
import datetime
import logging
from mailman.config import config
from mailman.core.errors import MailmanError
......@@ -38,6 +40,7 @@ from mailman.interfaces.autorespond import ResponseAction
from mailman.interfaces.bans import IBanManager
from mailman.interfaces.bounce import UnrecognizedBounceDisposition
from mailman.interfaces.digests import DigestFrequency
from mailman.interfaces.chain import LinkAction
from mailman.interfaces.languages import ILanguageManager
from mailman.interfaces.mailinglist import IAcceptableAliasSet
from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
......@@ -51,6 +54,8 @@ from sqlalchemy import Boolean
from urllib.error import URLError
from zope.component import getUtility
log = logging.getLogger('mailman.error')
class Import21Error(MailmanError):
......@@ -124,6 +129,24 @@ def nonmember_action_mapping(value):
3: Action.discard,
}[value]
def action_to_chain(value):
# Converts an action number in Mailman 2.1 to the name of the corresponding
# chain in 3.x. The actions "approve", "subscribe" and "unsubscribe" are
# ignored. The defer action is converted to None, because it is not a jump
# to a terminal chain.
return {
0: None,
#1: "approve",
2: "reject",
3: "discard",
#4: "subscribe",
#5: "unsubscribe",
6: "accept",
7: "hold",
}[value]
def check_language_code(code):
if code is None:
......@@ -310,6 +333,42 @@ def import_config_pck(mlist, config_dict):
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle header_filter_rules conversion to header_matches
header_matches = []
for line_patterns, action, _unused in \
config_dict.get('header_filter_rules', []):
chain = action_to_chain(action)
# now split the pattern in a header and a pattern
for line_pattern in line_patterns.splitlines():
if not line_pattern.strip():
continue
for sep in (': ', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
break # found it.
else:
# matches any header. Those are not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip("^").lower()
header = header.replace('\\', '')
if not header:
log.warning('Can\'t parse the header in header_filter_rule: %r',
line_pattern)
continue
if not pattern:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning('Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
header_matches.append((header, pattern, chain))
mlist.header_matches = header_matches
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
......
......@@ -50,6 +50,7 @@ from mailman.interfaces.nntp import NewsgroupModeration
from mailman.interfaces.templates import ITemplateLoader
from mailman.interfaces.usermanager import IUserManager
from mailman.testing.layers import ConfigLayer
from mailman.testing.helpers import LogFileMark
from mailman.utilities.filesystem import makedirs
from mailman.utilities.importer import import_config_pck, Import21Error
from mailman.utilities.string import expand
......@@ -330,6 +331,114 @@ class TestBasicImport(unittest.TestCase):
self.assertEqual(self._mlist.subscription_policy,
SubscriptionPolicy.confirm_then_moderate)
def test_header_matches(self):
# This test contail real cases of header_filter_rules
self._pckdict['header_filter_rules'] = [
('^X-Spam-Status: Yes', 3, False),
('X-Spam-Status: Yes', 3, False),
('X\\-Spam\\-Status\\: Yes.*', 3, False),
('X-Spam-Status: Yes\r\n\r\n', 2, False),
('^X-Spam-Level: \\*\\*\\*.*$', 3, False),
('^X-Spam-Level:.\\*\\*\r\n^X-Spam:.\\Yes', 3, False),
('Subject: \\[SPAM\\].*', 3, False),
('^Subject: .*loan.*', 3, False),
('Original-Received: from *linkedin.com*\r\n', 3, False),
('X-Git-Module: rhq.*git', 6, False),
('Approved: verysecretpassword', 6, False),
('^Subject: dev-\r\n^Subject: staging-', 3, False),
('from: .*[email protected]\r\nfrom: .*@jw-express.com', 2, False),
('^Received: from smtp-.*\\.fedoraproject\\.org\r\n'
'^Received: from mx.*\\.redhat.com\r\n'
'^Resent-date:\r\n'
'^Resent-from:\r\n'
'^Resent-Message-ID:\r\n'
'^Resent-to:\r\n'
'^Subject: [^mtv]\r\n',
7, False),
('^Received: from fedorahosted\\.org.*by fedorahosted\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by fedorahosted\\.org',
6, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [
('x-spam-status', 'Yes', 'discard'),
('x-spam-status', 'Yes', 'discard'),
('x-spam-status', 'Yes.*', 'discard'),
('x-spam-status', 'Yes', 'reject'),
('x-spam-level', '\\*\\*\\*.*$', 'discard'),
('x-spam-level', '\\*\\*', 'discard'),
('x-spam', '\\Yes', 'discard'),
('subject', '\\[SPAM\\].*', 'discard'),
('subject', '.*loan.*', 'discard'),
('original-received', 'from *linkedin.com*', 'discard'),
('x-git-module', 'rhq.*git', 'accept'),
('approved', 'verysecretpassword', 'accept'),
('subject', 'dev-', 'discard'),
('subject', 'staging-', 'discard'),
('from', '.*[email protected]', 'reject'),
('from', '.*@jw-express.com', 'reject'),
('received', 'from smtp-.*\\.fedoraproject\\.org', 'hold'),
('received', 'from mx.*\\.redhat.com', 'hold'),
('resent-date', '.*', 'hold'),
('resent-from', '.*', 'hold'),
('resent-message-id', '.*', 'hold'),
('resent-to', '.*', 'hold'),
('subject', '[^mtv]', 'hold'),
('received', 'from fedorahosted\\.org.*by fedorahosted\\.org', 'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by hosted.*\\.fedoraproject\\.org', 'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by fedoraproject\\.org', 'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by fedorahosted\\.org', 'accept'),
])
loglines = error_log.read().strip()
self.assertEqual(len(loglines), 0)
def test_header_matches_header_only(self):
# Check that an empty pattern is skipped
self._pckdict['header_filter_rules'] = [
('SomeHeaderName', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_header_matches_anything(self):
# Check that an empty pattern is skipped
self._pckdict['header_filter_rules'] = [
('.*', 7, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_header_matches_invalid_re(self):
# Check that an empty pattern is skipped
self._pckdict['header_filter_rules'] = [
('SomeHeaderName: *invalid-re', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Skipping header_filter rule because of an invalid '
'regular expression', error_log.readline())
def test_header_matches_defer(self):
# Check that a defer action is properly converted.
self._pckdict['header_filter_rules'] = [
('^X-Spam-Status: Yes', 0, False),
]
self._import()
self.assertListEqual(self._mlist.header_matches, [
('x-spam-status', 'Yes', None),
])
class TestArchiveImport(unittest.TestCase):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment