From 22209f5595976f07d9e6b933d410a25e917bd28d Mon Sep 17 00:00:00 2001 From: Michel Bernier <mbernier@cofomo.com> Date: Fri, 5 May 2023 16:27:31 -0400 Subject: [PATCH 1/3] IDNA 2008 mailman Change Message Object, Models, migrations and Connection to handle smtp_utf8 and fallback --- .gitignore | 1 + requirements-docs.txt | 1 + setup.py | 1 + src/mailman/app/inject.py | 4 +- src/mailman/app/lifecycle.py | 1 - src/mailman/app/membership.py | 2 +- src/mailman/app/notifications.py | 2 +- src/mailman/app/subscriptions.py | 11 ++-- src/mailman/commands/cli_addmembers.py | 3 +- src/mailman/commands/cli_delmembers.py | 3 +- src/mailman/commands/cli_members.py | 3 +- src/mailman/commands/cli_syncmembers.py | 3 +- src/mailman/commands/eml_membership.py | 3 +- src/mailman/commands/eml_who.py | 2 +- src/mailman/core/switchboard.py | 3 +- ...add_ascii_email_and_domain_name_columns.py | 52 ++++++++++++++++++ src/mailman/database/types.py | 4 +- src/mailman/email/message.py | 10 ++-- src/mailman/email/tests/test_validate.py | 24 ++++++++ src/mailman/email/validate.py | 24 +++----- src/mailman/handlers/avoid_duplicates.py | 3 +- src/mailman/handlers/cleanse.py | 3 +- src/mailman/handlers/cook_headers.py | 3 +- src/mailman/handlers/decorate.py | 2 +- src/mailman/handlers/dmarc.py | 3 +- src/mailman/handlers/rfc_2369.py | 2 +- src/mailman/handlers/subject_prefix.py | 4 +- src/mailman/interfaces/address.py | 3 + src/mailman/model/address.py | 19 ++++--- src/mailman/model/bans.py | 27 ++++++--- src/mailman/model/domain.py | 18 ++++-- src/mailman/model/listmanager.py | 6 +- src/mailman/model/mailinglist.py | 4 ++ src/mailman/model/roster.py | 6 +- src/mailman/model/tests/test_domain.py | 10 ++++ src/mailman/model/usermanager.py | 18 +++--- src/mailman/mta/connection.py | 40 +++++++++++--- src/mailman/mta/personalized.py | 2 +- src/mailman/mta/tests/test_connection.py | 13 ++--- src/mailman/rest/addresses.py | 1 + src/mailman/rest/lists.py | 2 - src/mailman/rest/validator.py | 1 - src/mailman/runners/digest.py | 2 +- src/mailman/runners/lmtp.py | 5 +- src/mailman/testing/helpers.py | 6 +- src/mailman/utilities/email.py | 29 +++++++++- src/mailman/utilities/tests/test_email.py | 29 +++++++++- src/mailman/utilities/tests/test_ua_utils.py | 55 +++++++++++++++++++ src/mailman/utilities/ua_utils.py | 49 +++++++++++++++++ tox.ini | 4 +- 50 files changed, 419 insertions(+), 107 deletions(-) create mode 100644 src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py create mode 100644 src/mailman/utilities/tests/test_ua_utils.py create mode 100644 src/mailman/utilities/ua_utils.py diff --git a/.gitignore b/.gitignore index d8fddc74b8..abecd7401d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dist venv *.mo *.po~ +*.iml diff --git a/requirements-docs.txt b/requirements-docs.txt index f8bbe853e7..e7bc3387b3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,3 +2,4 @@ sphinx>=3.2,!=5.2.0.post0 sphinx_rtd_theme docutils<0.18,>=0.14 sphinxcontrib-zopeext +email_validator diff --git a/setup.py b/setup.py index 5448b7cf95..50c2aaf701 100644 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ case second 'm'. Any other spelling is incorrect.""", 'zope.configuration', 'zope.event', 'zope.interface>=5.0', + 'email-validator>=1.3.0' ], ) diff --git a/src/mailman/app/inject.py b/src/mailman/app/inject.py index a289a816eb..c51a060fd1 100644 --- a/src/mailman/app/inject.py +++ b/src/mailman/app/inject.py @@ -17,7 +17,7 @@ """Inject a message into a queue.""" -from email import message_from_bytes +from email import message_from_bytes, policy from email.utils import formatdate, make_msgid from mailman.config import config from mailman.email.message import Message @@ -94,5 +94,5 @@ def inject_text(mlist, text, recipients=None, switchboard=None, **kws): if isinstance(text, str): text = text.encode('utf-8') assert isinstance(text, bytes), 'Bad text input to inject_text' - message = message_from_bytes(text, Message) + message = message_from_bytes(text, Message, policy=policy.default) return inject_message(mlist, message, recipients, switchboard, **kws) diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py index 0f586ebea0..99a9583a99 100644 --- a/src/mailman/app/lifecycle.py +++ b/src/mailman/app/lifecycle.py @@ -37,7 +37,6 @@ from mailman.utilities.modules import call_name from public import public from zope.component import getUtility - log = logging.getLogger('mailman.error') # These are the only characters allowed in list names. A more restrictive # class can be specified in config.mailman.listname_chars. diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index e982cb0aa3..82aabcaed5 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -17,7 +17,7 @@ """Application support for membership management.""" -from email.utils import formataddr +from mailman.utilities.email import formataddr from mailman.app.notifications import ( send_admin_subscription_notice, send_goodbye_message, diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 86007daedc..f9ee480a27 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -21,7 +21,7 @@ import logging from email.mime.message import MIMEMessage from email.mime.text import MIMEText -from email.utils import formataddr +from mailman.utilities.email import formataddr from lazr.config import as_boolean from mailman.config import config from mailman.core.i18n import _ diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 27d869c744..f9754e27f3 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -20,6 +20,7 @@ import uuid import logging +from mailman.utilities.email import formataddr from enum import Enum from lazr.config import as_boolean from mailman.app.membership import delete_member @@ -327,9 +328,8 @@ class SubscriptionWorkflow(_SubscriptionWorkflowCommon): subject = _( 'New subscription request to ${self.mlist.display_name} ' 'from ${self.address.email}') - username =\ - f'{self.subscriber.display_name} <{self.address.email}>'\ - if self.subscriber.display_name else self.address.email + username = formataddr( + (self.subscriber.display_name, self.address.email)) template = getUtility(ITemplateLoader).get( 'list:admin:action:subscribe', self.mlist) text = wrap(expand(template, self.mlist, dict( @@ -496,9 +496,8 @@ class UnSubscriptionWorkflow(_SubscriptionWorkflowCommon): subject = _( 'New unsubscription request to ${self.mlist.display_name} ' 'from ${self.address.email}') - username =\ - f'{self.subscriber.display_name} <{self.address.email}>'\ - if self.subscriber.display_name else self.address.email + username = formataddr( + (self.subscriber.display_name, self.address.email)) template = getUtility(ITemplateLoader).get( 'list:admin:action:unsubscribe', self.mlist) text = wrap(expand(template, self.mlist, dict( diff --git a/src/mailman/commands/cli_addmembers.py b/src/mailman/commands/cli_addmembers.py index 5fd748748a..0b1656bfcb 100644 --- a/src/mailman/commands/cli_addmembers.py +++ b/src/mailman/commands/cli_addmembers.py @@ -20,7 +20,8 @@ import sys import click -from email.utils import formataddr, parseaddr +from email.utils import parseaddr +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.database.transaction import transactional from mailman.interfaces.address import IEmailValidator diff --git a/src/mailman/commands/cli_delmembers.py b/src/mailman/commands/cli_delmembers.py index 60a12cfb49..82418b68af 100644 --- a/src/mailman/commands/cli_delmembers.py +++ b/src/mailman/commands/cli_delmembers.py @@ -20,7 +20,8 @@ import sys import click -from email.utils import formataddr, parseaddr +from email.utils import parseaddr +from mailman.utilities.email import formataddr from mailman.app.membership import delete_member from mailman.core.i18n import _ from mailman.database.transaction import transactional diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index d22d1af9c7..56d338c48f 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -19,6 +19,7 @@ import click +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager @@ -106,7 +107,7 @@ def display_members(ctx, mlist, role, regular, digest, if email_only or not address.display_name: print(address.original_email, file=outfp) else: - print(f'{address.display_name} <{address.original_email}>', + print(formataddr((address.display_name, address.original_email)), file=outfp) diff --git a/src/mailman/commands/cli_syncmembers.py b/src/mailman/commands/cli_syncmembers.py index 9547ef98ce..3d732a5979 100644 --- a/src/mailman/commands/cli_syncmembers.py +++ b/src/mailman/commands/cli_syncmembers.py @@ -20,7 +20,8 @@ import sys import click -from email.utils import formataddr, parseaddr +from email.utils import parseaddr +from mailman.utilities.email import formataddr from mailman.app.membership import delete_member from mailman.core.i18n import _ from mailman.database.transaction import transactional diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py index d3b90d2015..a6ee190852 100644 --- a/src/mailman/commands/eml_membership.py +++ b/src/mailman/commands/eml_membership.py @@ -18,7 +18,8 @@ """The email commands 'join' and 'subscribe'.""" from email.header import decode_header, make_header -from email.utils import formataddr, parseaddr +from email.utils import parseaddr +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.address import InvalidEmailAddressError from mailman.interfaces.command import ContinueProcessing, IEmailCommand diff --git a/src/mailman/commands/eml_who.py b/src/mailman/commands/eml_who.py index e536eae6af..c99916399f 100644 --- a/src/mailman/commands/eml_who.py +++ b/src/mailman/commands/eml_who.py @@ -19,7 +19,7 @@ import re -from email.utils import formataddr +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand from mailman.interfaces.member import DeliveryMode, DeliveryStatus diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index 4f1184c017..c6f909ac42 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -30,6 +30,7 @@ import email import pickle import hashlib import logging +from email import policy from mailman.config import config from mailman.email.message import Message @@ -159,7 +160,7 @@ class Switchboard: # have to generate the message later when we do size restriction # checking. original_size = len(msg) - msg = email.message_from_string(msg, Message) + msg = email.message_from_string(msg, Message, policy=policy.default) msg.original_size = original_size data['original_size'] = original_size return msg, data diff --git a/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py b/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py new file mode 100644 index 0000000000..e15c354a88 --- /dev/null +++ b/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py @@ -0,0 +1,52 @@ +# Copyright (C) 2020-2022 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <https://www.gnu.org/licenses/>. + +"""Add ascii email and domain name columns + +Revision ID: e46c5b3c876b +Revises: 98224512c9c2 +Create Date: 2022-09-23 14:09:48.711761 + +""" +from alembic import op +from sqlalchemy import Column, String + +# revision identifiers, used by Alembic. +revision = 'e46c5b3c876b' +down_revision = '98224512c9c2' + + +def upgrade(): + op.add_column('domain', Column('mail_host_alabel', String)) + op.create_index( + op.f('domain_mail_host_alabel_key'), 'domain', ['mail_host_alabel'], + unique=True) + op.add_column('address', Column('email_alabel', String)) + op.create_index( + op.f('ix_address_email_alabel'), 'address', ['mail_host_alabel'], + unique=True) + op.add_column('mailinglist', Column('mail_host_alabel', String)) + op.add_column('ban', Column('email_alabel', String)) + pass + + +def downgrade(): + op.drop_column('domain', Column('domain_host_alabel', String)) + op.drop_column('address', Column('email_alabel', String)) + op.drop_column('mailinglist', Column('mail_host_alabel', String)) + op.drop_column('ban', Column('email_alabel', String)) + pass diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 28a1a4632c..3ceba5243a 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -179,7 +179,7 @@ class SAUnicodeXL(TypeDecorator): @compiles(SAUnicodeXL, 'mysql') def compile_sa_unicode_xl(element, compiler, **kw): # We hardcode the collate here to make string comparison case sensitive. - return 'VARCHAR(20000) COLLATE utf8_bin' # pragma: nocover + return 'VARCHAR(20000) COLLATE utf8_bin' # pragma: nocover @compiles(SAUnicodeXL) @@ -206,4 +206,4 @@ def default_sa_text(element, compiler, **kw): @compiles(SAText, 'mysql') def compile_sa_text(element, compiler, **kw): # We hardcode the collate here to make string comparison case sensitive. - return 'TEXT COLLATE utf8mb4_bin' # pragma: nocover + return 'TEXT COLLATE utf8mb4_bin' # pragma: nocover diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index 221fab4f9b..cbc57f7f9e 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -40,7 +40,7 @@ COMMASPACE = ', ' @public -class Message(email.message.Message): +class Message(email.message.EmailMessage): # BAW: For debugging w/ bin/dumpdb. Apparently pprint uses repr. def __repr__(self): return self.__str__() @@ -52,9 +52,9 @@ class Message(email.message.Message): # Work around for https://bugs.python.org/issue27321 and # https://bugs.python.org/issue32330. try: - value = email.message.Message.as_string(self) + value = email.message.EmailMessage.as_string(self) except (KeyError, LookupError, UnicodeEncodeError): - value = email.message.Message.as_bytes(self).decode( + value = email.message.EmailMessage.as_bytes(self).decode( 'ascii', 'replace') # Also ensure no unicode surrogates in the returned string. return email.utils._sanitize(value) @@ -62,9 +62,9 @@ class Message(email.message.Message): def as_bytes(self): # Workaround for https://bugs.python.org/issue41307. try: - value = email.message.Message.as_bytes(self) + value = email.message.EmailMessage.as_bytes(self) except UnicodeEncodeError: - value = email.message.Message.as_string(self).encode('utf-8') + value = email.message.EmailMessage.as_string(self).encode('utf-8') return value @property diff --git a/src/mailman/email/tests/test_validate.py b/src/mailman/email/tests/test_validate.py index cd1ffc152d..2fb0023228 100644 --- a/src/mailman/email/tests/test_validate.py +++ b/src/mailman/email/tests/test_validate.py @@ -26,6 +26,19 @@ from mailman.interfaces.address import ( from mailman.testing.layers import ConfigLayer from zope.component import getUtility +parametrized_international_email = [ + ("食ã¹ã‚‹äºº@野èœã¨è‚‰.com",), + ("çà @testé.dé",), + ("المديلمنتدب@نامه‌ای.com",), + ("dÃ¥kiØ@faß.de",)] + +parametrized_invalid_email = [ + ("email.com",), + ("@email.com",), + ("invalid@email",), + ("invalid@",), + ("",) +] class TestValidate(unittest.TestCase): """Test email validation.""" @@ -58,3 +71,14 @@ class TestValidate(unittest.TestCase): self.assertRaises(InvalidEmailAddressError, self.validate, 'bad_address') + + def test_valid_email(self): + assert_that(self.validator.is_valid('valid@email.com')).is_true() + + @parameterized.expand(parametrized_international_email) + def test_valid_email_international(self, email): + assert_that(self.validator.is_valid(email)).is_true() + + @parameterized.expand(parametrized_invalid_email) + def test_invalid_email(self, invalid_email): + assert_that(self.validator.is_valid(invalid_email)).is_false() diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index a194a050ef..2ee1aa3333 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -1,4 +1,4 @@ -# Copyright (C) 2009-2023 by the Free Software Foundation, Inc. +# Copyright (C) 2009-2022 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -17,7 +17,9 @@ """Email address validation.""" -import re +from email_validator import EmailNotValidError, validate_email +from public import public +from zope.interface import implementer from mailman.interfaces.address import ( IEmailValidator, @@ -46,20 +48,12 @@ class Validator: """See `IEmailValidator`.""" if not email: return False - user, domain_parts = split_email(email) - # Strip quotes from quoted local. - user = re.sub('^"(.*)"$', r'\1', user) - if not user or len(_valid_local.sub('', user)) > 0: - return False - # Local, unqualified addresses are not allowed. - if not domain_parts: - return False - if len(domain_parts) < 2: + try: + # Validate the email address. + validate_email(email, check_deliverability=False) + return True + except Exception: return False - for p in domain_parts: - if len(p) == 0 or p[0] == '-' or len(_valid_domain.sub('', p)) > 0: - return False - return True def validate(self, email): """Validate an email address. diff --git a/src/mailman/handlers/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py index cf080291d8..833f2139d8 100644 --- a/src/mailman/handlers/avoid_duplicates.py +++ b/src/mailman/handlers/avoid_duplicates.py @@ -25,7 +25,8 @@ warning header, or pass it through, depending on the user's preferences. import re -from email.utils import formataddr, getaddresses +from email.utils import getaddresses +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from public import public diff --git a/src/mailman/handlers/cleanse.py b/src/mailman/handlers/cleanse.py index 7543643af1..49074108cd 100644 --- a/src/mailman/handlers/cleanse.py +++ b/src/mailman/handlers/cleanse.py @@ -20,7 +20,8 @@ import re import logging -from email.utils import formataddr, make_msgid +from email.utils import make_msgid +from mailman.utilities.email import formataddr from mailman.config import config from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index 57c80d9be9..33eba01589 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -20,7 +20,8 @@ import logging from email.header import Header -from email.utils import formataddr, getaddresses, parseaddr +from email.utils import getaddresses, parseaddr +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.mailinglist import Personalization, ReplyToMunging diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py index bc95760ba0..5603c6a56e 100644 --- a/src/mailman/handlers/decorate.py +++ b/src/mailman/handlers/decorate.py @@ -22,7 +22,7 @@ import copy import logging from email.mime.text import MIMEText -from email.utils import formataddr +from mailman.utilities.email import formataddr from mailman.archiving.mailarchive import MailArchive from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py index 95b1c9a3dc..0ec2b13db4 100644 --- a/src/mailman/handlers/dmarc.py +++ b/src/mailman/handlers/dmarc.py @@ -31,7 +31,8 @@ import logging from email.header import decode_header, Header from email.mime.message import MIMEMessage from email.mime.text import MIMEText -from email.utils import formataddr, getaddresses, make_msgid +from email.utils import getaddresses, make_msgid +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.mailinglist import DMARCMitigateAction, ReplyToMunging diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index bac9857cc0..d7229af7f5 100644 --- a/src/mailman/handlers/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -19,7 +19,7 @@ import logging -from email.utils import formataddr +from mailman.utilities.email import formataddr from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader from mailman.interfaces.archiver import ArchivePolicy diff --git a/src/mailman/handlers/subject_prefix.py b/src/mailman/handlers/subject_prefix.py index 577149cee4..94ca62fdb2 100644 --- a/src/mailman/handlers/subject_prefix.py +++ b/src/mailman/handlers/subject_prefix.py @@ -19,6 +19,8 @@ import re +import logging + from contextlib import suppress from email.header import decode_header, Header, make_header from mailman.core.i18n import _ @@ -202,5 +204,5 @@ class SubjectPrefix: mlist, msgdata, subject, prefix, prefix_pattern, ws) if new_subject is not None: del msg['subject'] - msg['Subject'] = new_subject + msg['subject'] = str(new_subject) return diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py index d2952b4325..3e5516c6bb 100644 --- a/src/mailman/interfaces/address.py +++ b/src/mailman/interfaces/address.py @@ -101,6 +101,9 @@ class IAddress(Interface): case preserved email address; `email` will always be lower case. """) + email_ascii = Attribute( + """transformed ascii email address""") + display_name = Attribute( """Optional display name associated with the email address.""") diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index ac3f0962f9..664f8390e8 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -16,8 +16,13 @@ # GNU Mailman. If not, see <https://www.gnu.org/licenses/>. """Model for addresses.""" +from public import public +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String +from sqlalchemy.orm import backref, relationship +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer -from email.utils import formataddr from mailman.database.model import Model from mailman.database.types import SAUnicode, SAUnicode4Byte from mailman.interfaces.address import ( @@ -26,12 +31,9 @@ from mailman.interfaces.address import ( IEmailValidator, ) from mailman.utilities.datetime import now -from public import public -from sqlalchemy import Column, DateTime, ForeignKey, Integer -from sqlalchemy.orm import backref, relationship -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer +from mailman.utilities.email import formataddr +from mailman.utilities.ua_utils import (transform_email_domain_to_ascii, + transform_email_to_utf8) @public @@ -43,6 +45,7 @@ class Address(Model): id = Column(Integer, primary_key=True) email = Column(SAUnicode, index=True, unique=True) + email_alabel = Column(String, index=True, unique=True) _original = Column(SAUnicode) display_name = Column(SAUnicode4Byte) _verified_on = Column('verified_on', DateTime) @@ -59,6 +62,8 @@ class Address(Model): getUtility(IEmailValidator).validate(email) lower_case = email.lower() self.email = lower_case + self.email_alabel = transform_email_domain_to_ascii(self.email) + self.email = transform_email_to_utf8(self.email_alabel) self.display_name = display_name self._original = (None if lower_case == email else email) self.registered_on = now() diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index b8d8dfc58f..4360970472 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -25,9 +25,11 @@ from mailman.database.types import SAUnicode from mailman.interfaces.bans import IBan, IBanManager from mailman.utilities.queries import QuerySequence from public import public -from sqlalchemy import Column, Integer, select +from sqlalchemy import Column, Integer, select, String, or_ from zope.interface import implementer +from mailman.utilities.ua_utils import transform_email_domain_to_ascii + @public @implementer(IBan) @@ -38,11 +40,13 @@ class Ban(Model): id = Column(Integer, primary_key=True) email = Column(SAUnicode, index=True) + email_alabel = Column(String, index=True) list_id = Column(SAUnicode, index=True) def __init__(self, email, list_id): super().__init__() self.email = email + self.email_alabel = transform_email_domain_to_ascii(email) self.list_id = list_id @@ -58,7 +62,9 @@ class BanManager: @dbconnection def ban(self, store, email): """See `IBanManager`.""" - bans = store.query(Ban).filter_by(email=email, list_id=self._list_id) + bans = store.query(Ban).filter(or_(Ban.email == email, + Ban.email_alabel == email), + Ban.list_id == self._list_id) if bans.count() == 0: ban = Ban(email, self._list_id) store.add(ban) @@ -66,8 +72,9 @@ class BanManager: @dbconnection def unban(self, store, email): """See `IBanManager`.""" - ban = store.query(Ban).filter_by( - email=email, list_id=self._list_id).first() + ban = store.query(Ban).filter(or_(Ban.email == email, + Ban.email_alabel == email), + Ban.list_id == self._list_id).first() if ban is not None: store.delete(ban) @@ -78,7 +85,9 @@ class BanManager: if list_id is None: # The client is asking for global bans. Look up bans on the # specific email address first. - bans = store.query(Ban).filter_by(email=email, list_id=None) + bans = store.query(Ban).filter(or_(Ban.email == email, + Ban.email_alabel == email), + Ban.list_id == None) if bans.count() > 0: return True # And now look for global pattern bans. @@ -89,11 +98,15 @@ class BanManager: return True else: # This is a list-specific ban. - bans = store.query(Ban).filter_by(email=email, list_id=list_id) + bans = store.query(Ban).filter(or_(Ban.email == email, + Ban.email_alabel == email), + Ban.list_id == list_id) if bans.count() > 0: return True # Try global bans next. - bans = store.query(Ban).filter_by(email=email, list_id=None) + bans = store.query(Ban).filter(or_(Ban.email == email, + Ban.email_alabel == email), + Ban.list_id == None) if bans.count() > 0: return True # Now try specific mailing list bans, but with a pattern. diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index c64d648e30..d2067d858d 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -33,12 +33,15 @@ from mailman.interfaces.user import IUser from mailman.interfaces.usermanager import IUserManager from mailman.model.mailinglist import MailingList from public import public -from sqlalchemy import Column, func, Integer, select +from sqlalchemy import Column, func, Integer, select, String, or_ from sqlalchemy.orm import relationship from zope.component import getUtility from zope.event import notify from zope.interface import implementer +from mailman.utilities.ua_utils import (convert_to_alabel, + validate_and_transform_to_utf8) + @public @implementer(IDomain) @@ -50,6 +53,7 @@ class Domain(Model): id = Column(Integer, primary_key=True) mail_host = Column(SAUnicode, unique=True) + mail_host_alabel = Column(String, unique=True) description = Column(SAUnicode4Byte) owners = relationship('User', secondary='domain_owner', @@ -59,7 +63,8 @@ class Domain(Model): def __init__(self, mail_host, description=None, owners=None, - alias_domain=None): + alias_domain=None, + idna_domain=None): """Create and register a domain. :param mail_host: The host name for the email interface. @@ -71,7 +76,8 @@ class Domain(Model): :param alias_domain: Alternate domain for Postfix :type alias_domain: string """ - self.mail_host = mail_host + self.mail_host = validate_and_transform_to_utf8(mail_host) + self.mail_host_alabel = convert_to_alabel(mail_host) self.description = description if owners is not None: self.add_owners(owners) @@ -83,7 +89,7 @@ class Domain(Model): """See `IDomain`.""" yield from store.query(MailingList).filter( MailingList.mail_host == self.mail_host - ).order_by(MailingList._list_id) + ).order_by(MailingList._list_id) def __repr__(self): """repr(a_domain)""" @@ -158,7 +164,9 @@ class DomainManager: @dbconnection def get(self, store, mail_host, default=None): """See `IDomainManager`.""" - domains = store.query(Domain).filter_by(mail_host=mail_host) + domains = store.query(Domain).filter(or_( + Domain.mail_host == mail_host, + Domain.mail_host_alabel == mail_host)) if domains.count() < 1: return default assert domains.count() == 1, ( diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index 09b3297f63..477f96af73 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -43,6 +43,8 @@ from sqlalchemy import select from zope.event import notify from zope.interface import implementer +from mailman.utilities.ua_utils import validate_and_transform_to_utf8 + @public @implementer(IListManager) @@ -53,9 +55,11 @@ class ListManager: def create(self, store, fqdn_listname): """See `IListManager`.""" fqdn_listname = fqdn_listname.lower() - listname, at, hostname = fqdn_listname.partition('@') + listname, hostname = fqdn_listname.rsplit('@', 1) + listname = listname.strip("'") if len(hostname) == 0: raise InvalidEmailAddressError(fqdn_listname) + hostname = validate_and_transform_to_utf8(hostname) list_id = '{}.{}'.format(listname, hostname) notify(ListCreatingEvent(fqdn_listname)) mlist = store.query(MailingList).filter_by(_list_id=list_id).first() diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index f54a6b6cad..d4fe042526 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -81,6 +81,7 @@ from sqlalchemy import ( LargeBinary, PickleType, select, + String, ) from sqlalchemy.event import listen from sqlalchemy.ext.hybrid import hybrid_property @@ -91,6 +92,7 @@ from zope.component import getUtility from zope.event import notify from zope.interface import implementer +from mailman.utilities.ua_utils import convert_to_alabel SPACE = ' ' UNDERSCORE = '_' @@ -111,6 +113,7 @@ class MailingList(Model): # List identity list_name = Column(SAUnicode, index=True) mail_host = Column(SAUnicode, index=True) + mail_host_alabel = Column(String, index=True) _list_id = Column('list_id', SAUnicode, index=True, unique=True) allow_list_posts = Column(Boolean) include_rfc2369_headers = Column(Boolean) @@ -230,6 +233,7 @@ class MailingList(Model): assert hostname, 'Bad list name: {0}'.format(fqdn_listname) self.list_name = listname self.mail_host = hostname + self.mail_host_alabel = convert_to_alabel(self.mail_host) self._list_id = '{0}.{1}'.format(listname, hostname) # For the pending database self.next_request_id = 1 diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index f7c4c54c72..984c228d86 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -104,14 +104,16 @@ class AbstractRoster: members_a = store.query(Member).filter( Member.list_id == self._mlist.list_id, Member.role == self.role, - Address.email == email, + or_(Address.email == email, + Address.email_alabel == email), Member.address_id == Address.id) # Here's a query that finds all members subscribed with their # preferred address. members_u = store.query(Member).filter( Member.list_id == self._mlist.list_id, Member.role == self.role, - Address.email == email, + or_(Address.email == email, + Address.email_alabel == email), Member.user_id == User.id, User._preferred_address_id == Address.id) return members_a.union(members_u).all() diff --git a/src/mailman/model/tests/test_domain.py b/src/mailman/model/tests/test_domain.py index a6afdec389..79e6f69967 100644 --- a/src/mailman/model/tests/test_domain.py +++ b/src/mailman/model/tests/test_domain.py @@ -181,6 +181,16 @@ class TestDomainManager(unittest.TestCase): self.assertEqual([owner.addresses[0].email for owner in domain.owners], ['anne@example.org', 'bart@example.net']) + def test_add_domain_utf8(self): + domain = self._manager.add('例題.com') + self.assertEqual(domain.mail_host_alabel, 'xn--fsqr78o.com') + self.assertEqual(domain.mail_host, '例題.com') + + def test_add_domain_alabel(self): + domain = self._manager.add('xn--fsqr78o.com') + self.assertEqual(domain.mail_host, '例題.com') + self.assertEqual(domain.mail_host_alabel, 'xn--fsqr78o.com') + class TestDomain(unittest.TestCase): layer = ConfigLayer diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py index 6d5fac2f58..a8b46f87c3 100644 --- a/src/mailman/model/usermanager.py +++ b/src/mailman/model/usermanager.py @@ -16,7 +16,6 @@ # GNU Mailman. If not, see <https://www.gnu.org/licenses/>. """A user manager.""" - from mailman.database.transaction import dbconnection from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager @@ -31,7 +30,6 @@ from public import public from sqlalchemy import or_, select from zope.interface import implementer - @public @implementer(IUserManager) class UserManager: @@ -108,7 +106,9 @@ class UserManager: @dbconnection def create_address(self, store, email, display_name=None): """See `IUserManager`.""" - addresses = store.query(Address).filter(Address.email == email.lower()) + addresses = store.query(Address).filter(or_( + Address.email == email.lower(), + Address.email_alabel == email.lower())) if addresses.count() == 1: found = addresses[0] raise ExistingAddressError(found.original_email) @@ -142,8 +142,9 @@ class UserManager: @dbconnection def get_address(self, store, email): """See `IUserManager`.""" - return store.query( - Address).filter_by(email=email.lower()).one_or_none() + return store.query(Address).filter( + or_(Address.email == email.lower(), + Address.email_alabel == email.lower())).one_or_none() @property @dbconnection @@ -170,6 +171,7 @@ class UserManager: q = '%{}%'.format(query) yield from store.query(User).join( Address, Address.user_id == User.id).filter( - or_(User.display_name.ilike(q), - Address.display_name.ilike(q), - Address.email.ilike(q))) + or_(User.display_name.ilike(q), + Address.display_name.ilike(q), + Address.email.ilike(q), + Address.email_alabel == q)) diff --git a/src/mailman/mta/connection.py b/src/mailman/mta/connection.py index cd4759a069..25c90635d1 100644 --- a/src/mailman/mta/connection.py +++ b/src/mailman/mta/connection.py @@ -24,12 +24,14 @@ import logging import smtplib from contextlib import suppress -from email.message import Message +from email.message import EmailMessage from lazr.config import as_boolean from mailman.config import config from mailman.interfaces.configuration import InvalidConfigurationError from public import public +from mailman.utilities.email import split_email +from mailman.utilities.ua_utils import transform_email_domain_to_ascii log = logging.getLogger('mailman.smtp') @@ -58,6 +60,7 @@ def as_SecureMode(s): class Connection: """Manage a connection to the SMTP server.""" + def __init__(self, host, port, sessions_per_connection, smtp_user=None, smtp_pass=None, secure_mode=SecureMode.INSECURE, @@ -135,16 +138,17 @@ class Connection: # object is preferred because of the treatment of line endings. if isinstance(msg, str): msg = msg.encode('ascii', 'replace') - assert isinstance(msg, bytes) or isinstance(msg, Message), \ + assert isinstance(msg, bytes) or isinstance(msg, EmailMessage), \ 'Connection.sendmail received an invalid msg arg.' try: - log.debug('envsender: %s, recipients: %s, size(msg): %s', - envsender, recipients, len(msg)) - if isinstance(msg, Message): - results = self._connection.send_message(msg, envsender, - recipients) - else: - results = self._connection.sendmail(envsender, recipients, msg) + results = self._sendmail(envsender, recipients, msg) + except smtplib.SMTPNotSupportedError as e: + recipients_ascii = list( + filter(lambda recipient: recipient is not None, + map(self._smtputf8_fallback, recipients))) + if len(recipients_ascii) != len(recipients): + raise e + results = self.sendmail(envsender, recipients, msg) except smtplib.SMTPException: # For safety, close this connection. The next send attempt will # automatically re-open it. Pass the exception on up. @@ -159,6 +163,24 @@ class Connection: self.quit() return results + def _smtputf8_fallback(self, recipient): + # first validate that the local_part is ascii, + # punycode being applied only to domain names + # encoding local part may result in errors. + local_part, domain = split_email(recipient) + if not local_part.isascii(): + return + return transform_email_domain_to_ascii(recipient) + + def _sendmail(self, envsender, recipients, msg): + log.debug('envsender: %s, recipients: %s, size(msg): %s', + envsender, recipients, len(msg)) + if isinstance(msg, EmailMessage): + return self._connection.send_message(msg, envsender, + recipients) + else: + return self._connection.sendmail(envsender, recipients, msg) + def quit(self): """Mimic `smtplib.SMTP.quit`.""" if self._connection is None: diff --git a/src/mailman/mta/personalized.py b/src/mailman/mta/personalized.py index 0dfebef460..a477b9b558 100644 --- a/src/mailman/mta/personalized.py +++ b/src/mailman/mta/personalized.py @@ -18,7 +18,7 @@ """Personalized delivery.""" from email.header import Header -from email.utils import formataddr +from mailman.utilities.email import formataddr from mailman.interfaces.mailinglist import Personalization from mailman.interfaces.usermanager import IUserManager from mailman.mta.verp import VERPDelivery diff --git a/src/mailman/mta/tests/test_connection.py b/src/mailman/mta/tests/test_connection.py index b0aa510944..7c1d9b1002 100644 --- a/src/mailman/mta/tests/test_connection.py +++ b/src/mailman/mta/tests/test_connection.py @@ -18,6 +18,8 @@ """Test MTA connections.""" import unittest +from smtplib import SMTPAuthenticationError, SMTPNotSupportedError +from unittest.mock import patch from mailman.config import config from mailman.mta.connection import Connection, SecureMode @@ -26,8 +28,6 @@ from mailman.testing.helpers import ( specialized_message_from_string as mfs, ) from mailman.testing.layers import SMTPLayer, SMTPSLayer, STARTTLSLayer -from smtplib import SMTPAuthenticationError, SMTPNotSupportedError -from unittest.mock import patch class TestConnection(unittest.TestCase): @@ -156,7 +156,6 @@ Subject: aardvarks class TestSMTPConnectionCount(_ConnectionCounter, unittest.TestCase): - layer = SMTPLayer def setUp(self): @@ -174,7 +173,6 @@ class TestSMTPConnectionCount(_ConnectionCounter, unittest.TestCase): class TestSMTPSConnectionCount(_ConnectionCounter, unittest.TestCase): - layer = SMTPSLayer def setUp(self): @@ -182,11 +180,10 @@ class TestSMTPSConnectionCount(_ConnectionCounter, unittest.TestCase): self.connection = Connection( config.mta.smtp_host, int(config.mta.smtp_port), 0, secure_mode=SecureMode.IMPLICIT, verify_cert=False - ) + ) class TestSTARTTLSConnectionCount(_ConnectionCounter, unittest.TestCase): - layer = STARTTLSLayer def setUp(self): @@ -194,7 +191,7 @@ class TestSTARTTLSConnectionCount(_ConnectionCounter, unittest.TestCase): self.connection = Connection( config.mta.smtp_host, int(config.mta.smtp_port), 0, secure_mode=SecureMode.STARTTLS, verify_cert=False - ) + ) class TestSTARTTLSNotSupported(unittest.TestCase): @@ -204,7 +201,7 @@ class TestSTARTTLSNotSupported(unittest.TestCase): connection = Connection( config.mta.smtp_host, int(config.mta.smtp_port), 0, secure_mode=SecureMode.STARTTLS, verify_cert=False - ) + ) msg_text = """\ From: anne@example.com To: bart@example.com diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index 15a8026b60..f6ad19a674 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -55,6 +55,7 @@ class _AddressBase(CollectionMixin): # email address. representation = dict( email=address.email, + email_alabel=address.email_alabel, original_email=address.original_email, registered_on=address.registered_on, self_link=self.api.path_to('addresses/{}'.format(address.email)), diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index d592ad62cb..38a6d1e168 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -16,7 +16,6 @@ # GNU Mailman. If not, see <https://www.gnu.org/licenses/>. """REST for mailing lists.""" - from lazr.config import as_boolean from mailman.app.digests import ( bump_digest_number_and_volume, @@ -65,7 +64,6 @@ from mailman.rest.validator import ( from public import public from zope.component import getUtility - def member_matcher(segments): """A matcher of member URLs inside mailing lists. diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index 1a82cef3cc..07a7d80508 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -16,7 +16,6 @@ # GNU Mailman. If not, see <https://www.gnu.org/licenses/>. """REST web form validation.""" - import re from lazr.config import as_boolean diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 8927f79b91..277df4e88b 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -60,7 +60,7 @@ class Digester: self._message = self._make_message() self._digest_part = self._make_digest_part() self._message['From'] = mlist.request_address - self._message['Subject'] = self._subject + self._message['Subject'] = str(self._subject) self._message['To'] = mlist.posting_address self._message['Reply-To'] = mlist.posting_address self._message['Date'] = formatdate(localtime=True) diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index 804c79a141..d36def9127 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -37,6 +37,7 @@ so that the peer mail server can provide better diagnostics. import re import email import logging +from email import policy from aiosmtpd.controller import Controller from aiosmtpd.lmtp import LMTP @@ -182,7 +183,9 @@ class LMTPHandler: listnames = set(getUtility(IListManager).names) # Parse the message data. If there are any defects in the # message, reject it right away; it's probably spam. - msg = email.message_from_bytes(envelope.content, Message) + msg = email.message_from_bytes(envelope.content, + Message, + policy=policy.default) msg.set_unixfrom(envelope.mail_from) except Exception: elog.exception('LMTP message parsing') diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 87f591f29a..83e77c7c09 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -30,7 +30,7 @@ import datetime import threading from contextlib import contextmanager, suppress -from email import message_from_bytes, message_from_string +from email import message_from_bytes, message_from_string, policy from lazr.config import as_timedelta from mailman.bin.master import Loop as Master from mailman.config import config @@ -255,7 +255,7 @@ def get_nntp_server(cleanups): class NNTPProxy: # noqa: E306 def get_message(self): args = nntpd.post.call_args - return message_from_bytes(args[0][0].read()) + return message_from_bytes(args[0][0].read(), policy=policy.default) return NNTPProxy() @@ -524,7 +524,7 @@ def specialized_message_from_string(unicode_text): # This mimic what Switchboard.dequeue() does when parsing a message from # text into a Message instance. original_size = len(unicode_text) - message = message_from_string(unicode_text, Message) + message = message_from_string(unicode_text, Message, policy=policy.default) message.original_size = original_size return message diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py index af6b7ccdab..c90acc94bf 100644 --- a/src/mailman/utilities/email.py +++ b/src/mailman/utilities/email.py @@ -16,11 +16,16 @@ # GNU Mailman. If not, see <https://www.gnu.org/licenses/>. """Email helpers.""" - +import re from base64 import b32encode +from email import utils from hashlib import sha1 + from public import public +specialsre = re.compile(r'[][\\()<>@,:;".]') +escapesre = re.compile(r'[\\"]') + @public def split_email(address): @@ -75,3 +80,25 @@ def add_message_hash(msg): del msg['x-message-id-hash'] msg['X-Message-ID-Hash'] = hash32 return hash32 + + +@public +def formataddr(pair, charset='utf-8'): + name, address = pair + if address.isascii(): + return utils.formataddr(pair) + else: + return formataddr_utf8(pair) + + +def formataddr_utf8(pair, charset='utf-8'): + name, address = pair + if name: + return "%s <%s>" % (name, address) + else: + quotes = '' + if specialsre.search(name): + quotes = '"' + name = escapesre.sub(r'\\\g<0>', name) + return '%s%s%s <%s>' % (quotes, name, quotes, address) + return address diff --git a/src/mailman/utilities/tests/test_email.py b/src/mailman/utilities/tests/test_email.py index 798cdbc71a..c4ac86874f 100644 --- a/src/mailman/utilities/tests/test_email.py +++ b/src/mailman/utilities/tests/test_email.py @@ -19,11 +19,15 @@ import unittest +from assertpy import assert_that +from parameterized import parameterized +from zope.component import getUtility + +from mailman.email.tests.test_validate import parametrized_international_email from mailman.interfaces.messages import IMessageStore from mailman.testing.helpers import specialized_message_from_string as mfs from mailman.testing.layers import ConfigLayer -from mailman.utilities.email import add_message_hash, split_email -from zope.component import getUtility +from mailman.utilities.email import add_message_hash, split_email, formataddr class TestEmail(unittest.TestCase): @@ -154,3 +158,24 @@ X-Message-ID-Hash: abc self.assertEqual(len(x_message_id_hashes), 1) self.assertEqual(x_message_id_hashes[0], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + + +class TestFormatAddr(unittest.TestCase): + + def test_formataddr_when_email_ascii(self): + address = "test@example.com" + name = "test tester" + result = formataddr((name, address)) + assert_that(result).is_equal_to("test tester <test@example.com>") + + @parameterized.expand(parametrized_international_email) + def test_formataddr_on_valid_utf8_email_ascii_username(self, email): + name = 'test tester' + result = formataddr((name, email)) + assert_that(result).is_equal_to("test tester <{}>".format(email)) + + @parameterized.expand(parametrized_international_email) + def test_formataddr_on_valid_email_utf8_username(self, email): + name = 'ã—ã’る タナカ' + result = formataddr((name, email)) + assert_that(result).is_equal_to("ã—ã’る タナカ <{}>".format(email)) diff --git a/src/mailman/utilities/tests/test_ua_utils.py b/src/mailman/utilities/tests/test_ua_utils.py new file mode 100644 index 0000000000..8186762240 --- /dev/null +++ b/src/mailman/utilities/tests/test_ua_utils.py @@ -0,0 +1,55 @@ +import unittest + +import idna +from assertpy import assert_that +from parameterized import parameterized + +from mailman.utilities.ua_utils import (convert_to_alabel, + validate_and_transform_to_utf8, + transform_email_to_utf8, + transform_email_domain_to_ascii) + +parametrized_domain_name = [ + ("野èœã¨è‚‰.com", "xn--o9jw62v5cc81t.com", "野èœã¨è‚‰.com"), + ("testé.dé", "xn--test-epa.xn--d-bga", "testé.dé"), + ("المديلمنتدب.com", "xn--mgbcftb7jbgck5b.com", "المديلمنتدب.com"), + ("faß.de", "xn--fa-hia.de", "faß.de"), + ("例題.イオ", "xn--fsqr78o.xn--eckm", "例題.イオ"), + ("例題.xn--eckm", "xn--fsqr78o.xn--eckm", "例題.イオ") +] + +parametrized_international_email = [ + ("食ã¹ã‚‹äºº@野èœã¨è‚‰.com", "食ã¹ã‚‹äºº@xn--o9jw62v5cc81t.com"), + ("çà @testé.dé", "çà @xn--test-epa.xn--d-bga"), + ("المديلمنتدب@نامه‌ای.com", "المديلمنتدب@xn--mgba3gch31f060k.com"), + ("dÃ¥kiØ@faß.de", "dÃ¥kiØ@xn--fa-hia.de")] + + +class TestDomainTranslate(unittest.TestCase): + + @parameterized.expand(parametrized_domain_name) + def test_normal_utf8_string(self, domain, aLabel, domain_utf8): + result = convert_to_alabel(domain) + assert_that(result).is_equal_to(aLabel) + + def test_too_long_domain_name(self): + long_domain_name = "x" * 64 + long_domain_name += ".com" + assert_that(convert_to_alabel).raises(idna.IDNAError).when_called_with( + long_domain_name) + + def test_too_short_domain_name(self): + assert_that(convert_to_alabel).raises(idna.IDNAError).when_called_with( + '') + + @parameterized.expand(parametrized_domain_name) + def test_convert_to_utf8(self, domain, alabel, domain_utf8): + assert_that(validate_and_transform_to_utf8(alabel)).is_equal_to(domain_utf8) + + @parameterized.expand(parametrized_international_email) + def test_transform_to_ascii(self, email, expected): + assert_that(transform_email_domain_to_ascii(email)).is_equal_to(expected) + + @parameterized.expand(parametrized_international_email) + def test_transform_email_to_ascii(self, expected, email): + assert_that(transform_email_to_utf8(email)).is_equal_to(expected) diff --git a/src/mailman/utilities/ua_utils.py b/src/mailman/utilities/ua_utils.py new file mode 100644 index 0000000000..8f764d1aae --- /dev/null +++ b/src/mailman/utilities/ua_utils.py @@ -0,0 +1,49 @@ +import logging + +import idna + +from mailman.utilities.email import split_email + +logger = logging.getLogger("mailman.core") + + +def convert_to_alabel(domain): + """ + Convert a domain name from U-Label to A-Label + :param domain: The domain name + :return the conversion result + """ + try: + # Encode the domain name in A-Label to be used by DNS, etc. client. + # IDNA package is fully IDNA 2008 compliant + domain_a_label = idna.encode(domain, uts46=True).decode('ascii') + logger.info( + f"Domain '{domain}' converted in A-Label is '{domain_a_label}'") + return domain_a_label + except idna.IDNAError as e: + # The label is invalid as per IDNA 2008 + logger.error(f"Domain '{domain}' is invalid: {e}") + raise e + except Exception as e: + # Unexpected exception + logger.error(f"ERROR: {e}") + # as this does not seem to be an IDNA related exception, always raise + raise e + + +def transform_email_domain_to_ascii(email): + local_part, domain = split_email(email) + domain_alabel = convert_to_alabel(".".join(domain)) + return '{}@{}'.format(local_part, domain_alabel) + + +def transform_email_to_utf8(email): + local_part, domain = split_email(email) + domain_utf8 = validate_and_transform_to_utf8(".".join(domain)) + return '{}@{}'.format(local_part, domain_utf8) + + +def validate_and_transform_to_utf8(domain): + if 'xn--' in domain: + return idna.decode(domain) + return domain diff --git a/tox.ini b/tox.ini index a453d5fd7c..ab9656dab6 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,8 @@ deps = pg: psycopg2-binary mysql: pymysql diffcov: diff_cover>=6.0 + email_validator + parameterized==0.8.1 passenv = MAILMAN_* PYTHON* @@ -84,4 +86,4 @@ ignore = port_me/* *.yml ignore-bad-ideas = - src/mailman/testing/* \ No newline at end of file + src/mailman/testing/* -- GitLab From 3023a6a40eed8469a4c4efadb9f5f0895c293ef4 Mon Sep 17 00:00:00 2001 From: Michel Bernier <mbernier@cofomo.com> Date: Fri, 5 May 2023 16:28:54 -0400 Subject: [PATCH 2/3] Change lmtp communication for using utf8. Change connection with smtp to transform every email in punycode upon configuration. Change mailbox to use EAI. --- src/mailman/app/tests/test_bounces.py | 12 +++++++ src/mailman/config/schema.cfg | 1 + src/mailman/model/tests/test_bans.py | 2 +- src/mailman/mta/connection.py | 2 ++ src/mailman/runners/lmtp.py | 2 +- src/mailman/runners/tests/test_lmtp.py | 17 +++++++++- src/mailman/utilities/mailbox.py | 45 +++++++++++++++++++++++++- tox.ini | 3 +- 8 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/mailman/app/tests/test_bounces.py b/src/mailman/app/tests/test_bounces.py index a48f84dc8a..906a5e01c6 100644 --- a/src/mailman/app/tests/test_bounces.py +++ b/src/mailman/app/tests/test_bounces.py @@ -23,6 +23,8 @@ import shutil import tempfile import unittest +from parameterized import parameterized + from mailman.app.bounces import ( bounce_message, maybe_forward, @@ -46,6 +48,8 @@ from mailman.testing.helpers import ( from mailman.testing.layers import ConfigLayer from zope.component import getUtility +from mailman.utilities.tests.test_ua_utils import parametrized_international_email + class TestVERP(unittest.TestCase): """Test header VERP detection.""" @@ -561,3 +565,11 @@ Subject: Ignore bounce_message(self._mlist, self._msg) # Nothing in the virgin queue means nothing's been bounced. get_queue_messages('virgin', expected_count=0) + + @parameterized.expand(parametrized_international_email) + def test_internationalized_sender(self, email, email_transformed): + self._msg['from'] = email + bounce_message(self._mlist, self._msg) + # Nothing in the virgin queue means nothing's been bounced. + get_queue_messages('virgin', expected_count=0) + diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index ae0f5c70a9..4ab14abf56 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -777,6 +777,7 @@ smtp_host: localhost smtp_port: 25 smtp_user: smtp_pass: +always_punycode: no # One of smtp/smtps/starttls, specifies the protocol Mailman will use when # connecting. Typically will correspond to smtp_port: 25 -> smtp, 465 -> smtps, diff --git a/src/mailman/model/tests/test_bans.py b/src/mailman/model/tests/test_bans.py index a53125177b..e3e2462cf7 100644 --- a/src/mailman/model/tests/test_bans.py +++ b/src/mailman/model/tests/test_bans.py @@ -62,4 +62,4 @@ class TestMailingListBans(unittest.TestCase): # The results can be indexed. self.assertEqual( [self._manager.bans[i].email for i in range(count)], - ['ant@example.com', 'bee@example.com', 'cat@example.com']) + ['ant@example.com', 'bee@example.com', 'cat@example.com']) \ No newline at end of file diff --git a/src/mailman/mta/connection.py b/src/mailman/mta/connection.py index 25c90635d1..366477824f 100644 --- a/src/mailman/mta/connection.py +++ b/src/mailman/mta/connection.py @@ -141,6 +141,8 @@ class Connection: assert isinstance(msg, bytes) or isinstance(msg, EmailMessage), \ 'Connection.sendmail received an invalid msg arg.' try: + if as_boolean(config.mta.always_punycode): + recipients = [transform_email_domain_to_ascii(recipient) for recipient in recipients] results = self._sendmail(envsender, recipients, msg) except smtplib.SMTPNotSupportedError as e: recipients_ascii = list( diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index d36def9127..a9425269fe 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -283,7 +283,7 @@ class LMTPHandler: class LMTPController(Controller): def factory(self): - server = LMTP(self.handler) + server = LMTP(self.handler, enable_SMTPUTF8=True) server.__ident__ = 'GNU Mailman LMTP runner 2.0' return server diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 73ab1e749c..5ee3020346 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -314,6 +314,21 @@ Subject: This should be accepted. items = get_queue_messages('in', expected_count=1) self.assertEqual(items[0].msgdata['listid'], 'test.example.com') + def test_mailing_list_with_utf8_local_part_sender(self): + with transaction(): + create_list('testing@example.com') + self.assertEqual(self._mlist.posting_address, 'testing@example.com') + self._lmtp.sendmail('raphaël@example.com', ['testing@example.com'], """\ +From: raphaël@example.com +To: testing@example.com +Message-ID: <ant> +Subject: This should be accepted. + +""") + # The message is in the incoming queue but not the command queue. + items = get_queue_messages('in', expected_count=1) + self.assertEqual(items[0].msgdata['listid'], 'testing.example.com') + class TestBugs(unittest.TestCase): """Test some LMTP related bugs.""" @@ -358,7 +373,7 @@ Message-ID: <alpha> # Local parts > 64 bytes should be accepted. with transaction(): create_list('longer_than_15_bytes@example.com') - recip = 'longer_than_15_bytes-confirm+{}@example.com'.format(40*'x') + recip = 'longer_than_15_bytes-confirm+{}@example.com'.format(40 * 'x') self._lmtp.sendmail('anne@example.com', [recip], """\ From: anne@example.com To: {} diff --git a/src/mailman/utilities/mailbox.py b/src/mailman/utilities/mailbox.py index 59ca06530c..7f23ccbf87 100644 --- a/src/mailman/utilities/mailbox.py +++ b/src/mailman/utilities/mailbox.py @@ -24,8 +24,12 @@ # for us is that it does no 'From' mangling. # mangling. -from mailbox import MMDF +from mailbox import MMDF, _mboxMMDF, _mboxMMDFMessage, linesep +from mailbox import _mboxMMDF from public import public +import email +import email.message +import email.generator @public @@ -41,3 +45,42 @@ class Mailbox(MMDF): self.unlock() # Don't suppress the exception. return False + + def get_message(self, key): + """Return a Message representation or raise a KeyError.""" + start, stop = self._lookup(key) + self._file.seek(start) + from_line = self._file.readline().replace(linesep, b'') + string = self._file.read(stop - self._file.tell()) + msg = self._message_factory(string.replace(linesep, b'\n')) + msg.set_from(from_line[5:].decode('utf-8')) + return msg + + def _install_message(self, message): + """Format a message and blindly write to self._file.""" + from_line = None + if isinstance(message, str): + message = self._string_to_bytes(message) + if isinstance(message, bytes) and message.startswith(b'From '): + newline = message.find(b'\n') + if newline != -1: + from_line = message[:newline] + message = message[newline + 1:] + else: + from_line = message + message = b'' + elif isinstance(message, _mboxMMDFMessage): + author = message.get_from().encode('utf-8') + from_line = b'From ' + author + elif isinstance(message, email.message.Message): + from_line = message.get_unixfrom() # May be None. + if from_line is not None: + from_line = from_line.encode('utf-8') + if from_line is None: + from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode() + start = self._file.tell() + self._file.write(from_line + linesep) + self._dump_message(message, self._file, self._mangle_from_) + stop = self._file.tell() + return (start, stop) + diff --git a/tox.ini b/tox.ini index ab9656dab6..f27e22f519 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py37,py38,py39,py310,py311}-{nocov,cov,diffcov}{,-mysql,-pg},qa +envlist = {py37,py38,py39,py310}-{nocov,cov,diffcov}{,-mysql,-pg},qa #recreate = True skip_missing_interpreters = True @@ -16,6 +16,7 @@ commands = #sitepackages = True usedevelop = True deps = + distutils flufl.testing>=0.8 nose2 cov,diffcov: coverage -- GitLab From 0ca7fee692126b7ed025ff5d85a0b10c240dc12b Mon Sep 17 00:00:00 2001 From: Michel Bernier <mbernier@cofomo.com> Date: Thu, 18 May 2023 08:38:38 -0400 Subject: [PATCH 3/3] Modify message for rest API. --- src/mailman/email/message.py | 5 +++-- src/mailman/email/validate.py | 12 ------------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index cbc57f7f9e..9219cea1e1 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -35,6 +35,7 @@ from mailman.interfaces.address import IEmailValidator from public import public from zope.component import getUtility +from mailman.utilities.ua_utils import transform_email_to_utf8 COMMASPACE = ', ' @@ -80,7 +81,7 @@ class Message(email.message.EmailMessage): for address in self.senders: # This could be None or the empty string. if address: - return address + return transform_email_to_utf8(address) return '' @property @@ -135,7 +136,7 @@ class Message(email.message.EmailMessage): sender = sender.decode('ascii') if not validator.is_valid(sender): continue - clean_senders.append(sender) + clean_senders.append(transform_email_to_utf8(sender)) return clean_senders diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index 2ee1aa3333..802431cff8 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -25,18 +25,6 @@ from mailman.interfaces.address import ( IEmailValidator, InvalidEmailAddressError, ) -from mailman.utilities.email import split_email -from public import public -from zope.interface import implementer - - -# What other characters should be allowed? -_valid_local = re.compile("[-0-9a-z!#$%&'*+./=?@_`{}~]", re.IGNORECASE) -# Strictly speaking, both ^ and | are allowed and others are allowed in quoted -# local parts, but this can open the door to certain web exploits so we don't -# allow them. -_valid_domain = re.compile('[-a-z0-9]', re.IGNORECASE) -# These are the only characters allowed in domain parts. @public -- GitLab