Commit 3af7ed0b authored by Mark Sapiro's avatar Mark Sapiro

Merge branch 'fix_677' into 'master'

Fixed html_to_plaintext to properly encode the new payload.

Closes #677

See merge request !600
parents fbec4615 94fbd7dd
Pipeline #116309170 passed with stages
in 10 minutes and 18 seconds
......@@ -11,7 +11,7 @@ Here is a history of user visible changes to Mailman.
3.3.1
=====
(2019-XX-XX)
(2020-XX-XX)
Bugs
----
......@@ -41,6 +41,8 @@ Bugs
(Closes #665)
* Residial bounces are now marked as processed. (Closes #669)
* Find members API now searches for the Member's display name too. (Closes #667)
* Content filtering ``convert_html_to_plaintext`` no longer creates messages
that can't be flattened as_bytes. (Closes #677)
REST
----
......
......@@ -234,8 +234,9 @@ name of the file containing the message payload to filter.
... process(mlist, msg, {})
>>> print(msg.as_string())
From: [email protected]
Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
Content-Type: text/plain
Content-Type: text/plain; charset="us-ascii"
X-Content-Filtered-By: Mailman/MimeDel ...
<BLANKLINE>
Converted text/html to text/plain
......@@ -301,8 +302,9 @@ so the entire inner ``multipart/mixed`` is discarded.
X-Content-Filtered-By: Mailman/MimeDel ...
<BLANKLINE>
--AAA
Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
Content-Type: text/plain
Content-Type: text/plain; charset="us-ascii"
<BLANKLINE>
Converted text/html to text/plain
Filename: ...
......
......@@ -274,7 +274,7 @@ def to_plaintext(msg):
resources.callback(shutil.rmtree, tempdir)
for subpart in typed_subpart_iterator(msg, 'text', 'html'):
filename = os.path.join(tempdir, '{}.html'.format(next(counter)))
ctype = msg.get_content_charset('utf-8')
ctype = msg.get_content_charset('us-ascii')
with open(filename, 'w', encoding='utf-8') as fp:
fp.write(subpart.get_payload(decode=True).decode(ctype,
errors='replace'))
......@@ -288,7 +288,7 @@ def to_plaintext(msg):
# Replace the payload of the subpart with the converted text
# and tweak the content type.
del subpart['content-transfer-encoding']
subpart.set_payload(stdout)
subpart.set_payload(stdout, charset=ctype)
subpart.set_type('text/plain')
changedp += 1
return changedp
......
......@@ -45,6 +45,9 @@ from zope.component import getUtility
@contextmanager
def dummy_script(arg=''):
exe = sys.executable
non_ascii = ''
if arg == 'non-ascii':
non_ascii = '‘...’'
extra = ''
if arg == 'scripterr':
extra = 'error'
......@@ -59,10 +62,10 @@ def dummy_script(arg=''):
import sys
if len(sys.argv) > 2:
sys.exit(1)
print('Converted text/html to text/plain')
print('Converted text/html to text/plain{}')
print('Filename:', sys.argv[1])
print(open(sys.argv[1]).readlines()[0])
""", file=fp)
""".format(non_ascii), file=fp)
config.push('dummy script', """\
[mailman]
html_to_plain_text_command = {exe} {script} {extra} $filename
......@@ -261,6 +264,29 @@ PGh0bWw+PGhlYWQ+PC9oZWFkPgo8Ym9keT48L2JvZHk+PC9odG1sPgo=
self.assertEqual(payload_lines[0], 'Converted text/html to text/plain')
self.assertEqual(payload_lines[2], '<html><head></head>')
def test_convert_html_to_plaintext_encodes_new_payload(self):
# Test that the converted payload with non-ascii is encoded.
msg = mfs("""\
From: [email protected]
Content-Type: text/html; charset=utf-8
Content-Transfer-Encoding: base64
MIME-Version: 1.0
Q29udmVydGVkIHRleHQvaHRtbCB0byB0ZXh0L3BsYWlu4oCYLi4u4oCZCg==
""")
process = config.handlers['mime-delete'].process
with dummy_script('non-ascii'):
process(self._mlist, msg, {})
self.assertEqual(msg['content-type'], 'text/plain; charset="utf-8"')
self.assertEqual(msg['content-transfer-encoding'], 'base64')
self.assertTrue(
msg['x-content-filtered-by'].startswith('Mailman/MimeDel'))
payload_lines = (
msg.get_payload(decode=True).decode('utf-8').splitlines())
self.assertEqual(payload_lines[0],
'Converted text/html to text/plain‘...’')
self.assertTrue(payload_lines[1].startswith('Filename'))
def test_convert_html_to_plaintext_error_return(self):
# Calling a script which returns an error status is properly logged.
msg = mfs("""\
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment