Commit f37d002f authored by Mark Sapiro's avatar Mark Sapiro
Browse files

Implement rule to hold replies with digest subject or masthead quote.

parent 16787201
......@@ -62,6 +62,7 @@ class BuiltInChain:
('max-size', LinkAction.defer, None),
('news-moderation', LinkAction.defer, None),
('no-subject', LinkAction.defer, None),
('digests', LinkAction.defer, None),
('suspicious-header', LinkAction.defer, None),
# Now if any of the above hit, jump to the hold chain.
('any', LinkAction.jump, 'hold'),
......
......@@ -104,6 +104,7 @@ built-in chain. No rules hit and so the message is accepted.
max-size
news-moderation
no-subject
digests
suspicious-header
However, when Anne's moderation action is set to `hold`, her post is held for
......
......@@ -74,6 +74,15 @@ html_to_plain_text_command: /usr/bin/lynx -dump $filename
# unpredictable.
listname_chars: [-_.0-9a-z]
# Should we hold posts to lists that have a digest subject or quote the
# digest masthead?
hold_digest: no
# If hold_digest is yes, this is the minimum number of non-blank digest
# masthead lines that are needed to be matched in the message for it to be
# considered as quoting the masthead.
masthead_threshold: 4
# These hooks are deprecated, but are kept here so as not to break existing
# configuration files. However, these hooks are not run. Define a plugin
# instead.
......
......@@ -275,7 +275,7 @@ This message will end up in the `pipeline` queue.
X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; emergency;
loop; banned-address; member-moderation; nonmember-moderation;
administrivia; implicit-dest; max-recipients; max-size;
news-moderation; no-subject; suspicious-header
news-moderation; no-subject; digests; suspicious-header
<BLANKLINE>
An important message.
<BLANKLINE>
......@@ -290,6 +290,7 @@ hit and all rules that have missed.
administrivia
approved
banned-address
digests
dmarc-mitigation
emergency
implicit-dest
......
......@@ -19,6 +19,12 @@ Bugs
* Increased the size of the data column in the workflowstate table.
(Closes #793)
New Features
------------
* There is a new setting ``hold_digest`` in the ``[mailman]`` section of
mailman.cfg. If this is set to ``yes``, posts with a digest like Subject:
header or which quote the digest masthead will be held for moderation.
3.3.2
=====
......
......@@ -19,10 +19,12 @@ You can also get all the values for a particular section, such as the
default_language: en
email_commands_max_lines: 10
filtered_messages_are_preservable: no
hold_digest: no
html_to_plain_text_command: /usr/bin/lynx -dump $filename
http_etag: ...
layout: testing
listname_chars: [-_.0-9a-z]
masthead_threshold: 4
noreply_address: noreply
pending_request_life: 3d
post_hook:
......
......@@ -41,9 +41,11 @@ class TestSystemConfiguration(unittest.TestCase):
default_language='en',
email_commands_max_lines='10',
filtered_messages_are_preservable='no',
hold_digest='no',
html_to_plain_text_command='/usr/bin/lynx -dump $filename',
layout='testing',
listname_chars='[-_.0-9a-z]',
masthead_threshold='4',
noreply_address='noreply',
pending_request_life='3d',
post_hook='',
......
# Copyright (C) 2007-2020 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <https://www.gnu.org/licenses/>.
"""The digest reply rule."""
import re
from lazr.config import as_boolean
from mailman.config import config
from mailman.core.i18n import _
from mailman.interfaces.rules import IRule
from mailman.interfaces.template import ITemplateLoader
from mailman.utilities.string import expand, wrap
from public import public
from zope.component import getUtility
from zope.interface import implementer
# Re to recognize a digest subject:
DIGRE = re.compile(r' Digest, Vol \d+, Issue \d+$', re.IGNORECASE)
@public
@implementer(IRule)
class Digests:
"""The digest reply rule."""
name = 'digests'
description = _('Catch messages with digest Subject or boilerplate quote.')
record = True
def check(self, mlist, msg, msgdata):
"""See `IRule`."""
if not as_boolean(config.mailman.hold_digest):
return False
# Convert the header value to a str because it may be an
# email.header.Header instance.
subject = str(msg.get('subject', '')).strip()
if DIGRE.search(subject):
msgdata['moderation_sender'] = msg.sender
with _.defer_translation():
# This will be translated at the point of use.
msgdata.setdefault('moderation_reasons', []).append(
_('Message has a digest subject'))
return True
# Get the masthead, but without emails.
mastheadtxt = getUtility(ITemplateLoader).get(
'list:member:digest:masthead', mlist)
mastheadtxt = wrap(expand(mastheadtxt, mlist, dict(
display_name=mlist.display_name,
listname='',
list_id=mlist.list_id,
request_email='',
owner_email='',
)))
msgtext = ''
for part in msg.walk():
if part.get_content_maintype() == 'text':
cset = part.get_content_charset('utf-8')
msgtext += part.get_payload(decode=True).decode(
cset, errors='replace')
matches = 0
lines = mastheadtxt.splitlines()
for line in lines:
line = line.strip()
if not line:
continue
if msgtext.find(line) >= 0:
matches += 1
if matches >= int(config.mailman.masthead_threshold):
msgdata['moderation_sender'] = msg.sender
with _.defer_translation():
# This will be translated at the point of use.
msgdata.setdefault('moderation_reasons', []).append(
_('Message quotes digest boilerplate'))
return True
return False
=======
Digests
=======
The ``digests`` rule matches when the posted message has a digest Subject:
header or quotes the digest masthead. Generally this is used to prevent
replies to a digest with no meaningful Subject: or which quote the entire
digest from getting posted to the list. This rule must be enabled by putting
``hold_digest: yes`` in the ``[mailman]`` section of the configuration.
>>> from mailman.app.lifecycle import create_list
>>> mlist = create_list('_xtest@example.com')
>>> from mailman.config import config
>>> rule = config.rules['digests']
>>> print(rule.name)
digests
If we enable the rule and post a message with a digest like Subject:, the
rule will hit.
>>> from mailman.testing.helpers import (
... configuration, specialized_message_from_string as message_from_string)
>>> msg = message_from_string("""\
... From: anne@example.com
... To: _xtest@example.com
... Subject: Re: test Digest, Vol 1, Issue 1
... Message-ID: <ant>
...
... A message body.
... """)
>>> with configuration('mailman', hold_digest='yes'):
... rule.check(mlist, msg, {})
True
Similarly, the rule will hit on a message with quotes of the digest masthead
regardless of the Subject:.
>>> msg = message_from_string("""\
... From: anne@example.com
... To: _xtest@example.com
... Subject: Message Subject
... Message-ID: <ant>
...
... Send _xtest mailing list submissions to
... _xtest@example.com
...
... To subscribe or unsubscribe via email, send a message with subject or body
... 'help' to
... _xtest-request@example.com
...
... You can reach the person managing the list at
... _xtest-owner@example.com
...
... When replying, please edit your Subject line so it is more specific than
... "Re: Contents of _xtest digest..."
... """)
>>> with configuration('mailman', hold_digest='yes'):
... rule.check(mlist, msg, {})
True
......@@ -23,6 +23,7 @@ names to rule objects.
any True
approved True
banned-address True
digests True
dmarc-mitigation True
emergency True
implicit-dest True
......
# Copyright (C) 2016-2020 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <https://www.gnu.org/licenses/>.
"""Test the `digests` rule."""
import unittest
from mailman.app.lifecycle import create_list
from mailman.rules import digests
from mailman.testing.helpers import (
configuration, specialized_message_from_string as mfs)
from mailman.testing.layers import ConfigLayer
class TestDigestsRule(unittest.TestCase):
"""Test the max_size rule."""
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('test@example.com')
@configuration('mailman', hold_digest='yes')
def test_digest_subject_reason(self):
# Ensure digests rule returns a reason for subject hit.
msg = mfs("""\
From: anne@example.com
To: test@example.com
Subject: Re: test Digest, Vol 1, Issue 1
Message-ID: <ant>
A message body.
""")
msgdata = {}
rule = digests.Digests()
result = rule.check(self._mlist, msg, msgdata)
self.assertTrue(result)
self.assertEqual(msgdata['moderation_reasons'],
['Message has a digest subject'])
@configuration('mailman', hold_digest='yes')
def test_digest_masthead_reason(self):
# Ensure digests rule returns a reason for masthead hit.
msg = mfs("""\
From: anne@example.com
To: test@example.com
Subject: Message Subject
Message-ID: <ant>
Send Test mailing list submissions to
test@example.com
To subscribe or unsubscribe via email, send a message with subject or body
'help' to
test-request@example.com
You can reach the person managing the list at
test-owner@example.com
When replying, please edit your Subject line so it is more specific than
"Re: Contents of $display_name digest..."
""")
msgdata = {}
rule = digests.Digests()
result = rule.check(self._mlist, msg, msgdata)
self.assertTrue(result)
self.assertEqual(msgdata['moderation_reasons'],
['Message quotes digest boilerplate'])
@configuration('mailman', hold_digest='yes')
def test_miss_on_ok_message(self):
# Rule should miss if not digest subject or masthead.
msg = mfs("""\
From: anne@example.com
To: test@example.com
Subject: Message Subject
Message-ID: <ant>
A message body.
""")
msgdata = {}
rule = digests.Digests()
result = rule.check(self._mlist, msg, msgdata)
self.assertFalse(result)
@configuration('mailman', hold_digest='no')
def test_no_hit_if_not_configured(self):
# Ensure rule misses if not configured.
msg = mfs("""\
From: anne@example.com
To: test@example.com
Subject: Re: test Digest, Vol 1, Issue 1
Message-ID: <ant>
Send Test mailing list submissions to
test@example.com
To subscribe or unsubscribe via email, send a message with subject or body
'help' to
test-request@example.com
You can reach the person managing the list at
test-owner@example.com
When replying, please edit your Subject line so it is more specific than
"Re: Contents of Test digest..."
""")
msgdata = {}
rule = digests.Digests()
result = rule.check(self._mlist, msg, msgdata)
self.assertFalse(result)
......@@ -134,7 +134,7 @@ Now the message is in the pipeline queue.
X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; emergency;
loop; banned-address; member-moderation; nonmember-moderation;
administrivia; implicit-dest; max-recipients; max-size;
news-moderation; no-subject; suspicious-header
news-moderation; no-subject; digests; suspicious-header
<BLANKLINE>
First post!
<BLANKLINE>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment