From 22209f5595976f07d9e6b933d410a25e917bd28d Mon Sep 17 00:00:00 2001
From: Michel Bernier <mbernier@cofomo.com>
Date: Fri, 5 May 2023 16:27:31 -0400
Subject: [PATCH 1/3] IDNA 2008 mailman

Change Message Object, Models, migrations and Connection to handle smtp_utf8 and fallback
---
 .gitignore                                    |  1 +
 requirements-docs.txt                         |  1 +
 setup.py                                      |  1 +
 src/mailman/app/inject.py                     |  4 +-
 src/mailman/app/lifecycle.py                  |  1 -
 src/mailman/app/membership.py                 |  2 +-
 src/mailman/app/notifications.py              |  2 +-
 src/mailman/app/subscriptions.py              | 11 ++--
 src/mailman/commands/cli_addmembers.py        |  3 +-
 src/mailman/commands/cli_delmembers.py        |  3 +-
 src/mailman/commands/cli_members.py           |  3 +-
 src/mailman/commands/cli_syncmembers.py       |  3 +-
 src/mailman/commands/eml_membership.py        |  3 +-
 src/mailman/commands/eml_who.py               |  2 +-
 src/mailman/core/switchboard.py               |  3 +-
 ...add_ascii_email_and_domain_name_columns.py | 52 ++++++++++++++++++
 src/mailman/database/types.py                 |  4 +-
 src/mailman/email/message.py                  | 10 ++--
 src/mailman/email/tests/test_validate.py      | 24 ++++++++
 src/mailman/email/validate.py                 | 24 +++-----
 src/mailman/handlers/avoid_duplicates.py      |  3 +-
 src/mailman/handlers/cleanse.py               |  3 +-
 src/mailman/handlers/cook_headers.py          |  3 +-
 src/mailman/handlers/decorate.py              |  2 +-
 src/mailman/handlers/dmarc.py                 |  3 +-
 src/mailman/handlers/rfc_2369.py              |  2 +-
 src/mailman/handlers/subject_prefix.py        |  4 +-
 src/mailman/interfaces/address.py             |  3 +
 src/mailman/model/address.py                  | 19 ++++---
 src/mailman/model/bans.py                     | 27 ++++++---
 src/mailman/model/domain.py                   | 18 ++++--
 src/mailman/model/listmanager.py              |  6 +-
 src/mailman/model/mailinglist.py              |  4 ++
 src/mailman/model/roster.py                   |  6 +-
 src/mailman/model/tests/test_domain.py        | 10 ++++
 src/mailman/model/usermanager.py              | 18 +++---
 src/mailman/mta/connection.py                 | 40 +++++++++++---
 src/mailman/mta/personalized.py               |  2 +-
 src/mailman/mta/tests/test_connection.py      | 13 ++---
 src/mailman/rest/addresses.py                 |  1 +
 src/mailman/rest/lists.py                     |  2 -
 src/mailman/rest/validator.py                 |  1 -
 src/mailman/runners/digest.py                 |  2 +-
 src/mailman/runners/lmtp.py                   |  5 +-
 src/mailman/testing/helpers.py                |  6 +-
 src/mailman/utilities/email.py                | 29 +++++++++-
 src/mailman/utilities/tests/test_email.py     | 29 +++++++++-
 src/mailman/utilities/tests/test_ua_utils.py  | 55 +++++++++++++++++++
 src/mailman/utilities/ua_utils.py             | 49 +++++++++++++++++
 tox.ini                                       |  4 +-
 50 files changed, 419 insertions(+), 107 deletions(-)
 create mode 100644 src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py
 create mode 100644 src/mailman/utilities/tests/test_ua_utils.py
 create mode 100644 src/mailman/utilities/ua_utils.py

diff --git a/.gitignore b/.gitignore
index d8fddc74b8..abecd7401d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ dist
 venv
 *.mo
 *.po~
+*.iml
diff --git a/requirements-docs.txt b/requirements-docs.txt
index f8bbe853e7..e7bc3387b3 100644
--- a/requirements-docs.txt
+++ b/requirements-docs.txt
@@ -2,3 +2,4 @@ sphinx>=3.2,!=5.2.0.post0
 sphinx_rtd_theme
 docutils<0.18,>=0.14
 sphinxcontrib-zopeext
+email_validator
diff --git a/setup.py b/setup.py
index 5448b7cf95..50c2aaf701 100644
--- a/setup.py
+++ b/setup.py
@@ -132,6 +132,7 @@ case second 'm'.  Any other spelling is incorrect.""",
         'zope.configuration',
         'zope.event',
         'zope.interface>=5.0',
+        'email-validator>=1.3.0'
         ],
     )
 
diff --git a/src/mailman/app/inject.py b/src/mailman/app/inject.py
index a289a816eb..c51a060fd1 100644
--- a/src/mailman/app/inject.py
+++ b/src/mailman/app/inject.py
@@ -17,7 +17,7 @@
 
 """Inject a message into a queue."""
 
-from email import message_from_bytes
+from email import message_from_bytes, policy
 from email.utils import formatdate, make_msgid
 from mailman.config import config
 from mailman.email.message import Message
@@ -94,5 +94,5 @@ def inject_text(mlist, text, recipients=None, switchboard=None, **kws):
     if isinstance(text, str):
         text = text.encode('utf-8')
     assert isinstance(text, bytes), 'Bad text input to inject_text'
-    message = message_from_bytes(text, Message)
+    message = message_from_bytes(text, Message, policy=policy.default)
     return inject_message(mlist, message, recipients, switchboard, **kws)
diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py
index 0f586ebea0..99a9583a99 100644
--- a/src/mailman/app/lifecycle.py
+++ b/src/mailman/app/lifecycle.py
@@ -37,7 +37,6 @@ from mailman.utilities.modules import call_name
 from public import public
 from zope.component import getUtility
 
-
 log = logging.getLogger('mailman.error')
 # These are the only characters allowed in list names.  A more restrictive
 # class can be specified in config.mailman.listname_chars.
diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py
index e982cb0aa3..82aabcaed5 100644
--- a/src/mailman/app/membership.py
+++ b/src/mailman/app/membership.py
@@ -17,7 +17,7 @@
 
 """Application support for membership management."""
 
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from mailman.app.notifications import (
     send_admin_subscription_notice,
     send_goodbye_message,
diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py
index 86007daedc..f9ee480a27 100644
--- a/src/mailman/app/notifications.py
+++ b/src/mailman/app/notifications.py
@@ -21,7 +21,7 @@ import logging
 
 from email.mime.message import MIMEMessage
 from email.mime.text import MIMEText
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from lazr.config import as_boolean
 from mailman.config import config
 from mailman.core.i18n import _
diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py
index 27d869c744..f9754e27f3 100644
--- a/src/mailman/app/subscriptions.py
+++ b/src/mailman/app/subscriptions.py
@@ -20,6 +20,7 @@
 import uuid
 import logging
 
+from mailman.utilities.email import formataddr
 from enum import Enum
 from lazr.config import as_boolean
 from mailman.app.membership import delete_member
@@ -327,9 +328,8 @@ class SubscriptionWorkflow(_SubscriptionWorkflowCommon):
             subject = _(
                 'New subscription request to ${self.mlist.display_name} '
                 'from ${self.address.email}')
-            username =\
-                f'{self.subscriber.display_name} <{self.address.email}>'\
-                if self.subscriber.display_name else self.address.email
+            username = formataddr(
+                (self.subscriber.display_name, self.address.email))
             template = getUtility(ITemplateLoader).get(
                 'list:admin:action:subscribe', self.mlist)
             text = wrap(expand(template, self.mlist, dict(
@@ -496,9 +496,8 @@ class UnSubscriptionWorkflow(_SubscriptionWorkflowCommon):
             subject = _(
                 'New unsubscription request to ${self.mlist.display_name} '
                 'from ${self.address.email}')
-            username =\
-                f'{self.subscriber.display_name} <{self.address.email}>'\
-                if self.subscriber.display_name else self.address.email
+            username = formataddr(
+                (self.subscriber.display_name, self.address.email))
             template = getUtility(ITemplateLoader).get(
                 'list:admin:action:unsubscribe', self.mlist)
             text = wrap(expand(template, self.mlist, dict(
diff --git a/src/mailman/commands/cli_addmembers.py b/src/mailman/commands/cli_addmembers.py
index 5fd748748a..0b1656bfcb 100644
--- a/src/mailman/commands/cli_addmembers.py
+++ b/src/mailman/commands/cli_addmembers.py
@@ -20,7 +20,8 @@
 import sys
 import click
 
-from email.utils import formataddr, parseaddr
+from email.utils import parseaddr
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.database.transaction import transactional
 from mailman.interfaces.address import IEmailValidator
diff --git a/src/mailman/commands/cli_delmembers.py b/src/mailman/commands/cli_delmembers.py
index 60a12cfb49..82418b68af 100644
--- a/src/mailman/commands/cli_delmembers.py
+++ b/src/mailman/commands/cli_delmembers.py
@@ -20,7 +20,8 @@
 import sys
 import click
 
-from email.utils import formataddr, parseaddr
+from email.utils import parseaddr
+from mailman.utilities.email import formataddr
 from mailman.app.membership import delete_member
 from mailman.core.i18n import _
 from mailman.database.transaction import transactional
diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py
index d22d1af9c7..56d338c48f 100644
--- a/src/mailman/commands/cli_members.py
+++ b/src/mailman/commands/cli_members.py
@@ -19,6 +19,7 @@
 
 import click
 
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.command import ICLISubCommand
 from mailman.interfaces.listmanager import IListManager
@@ -106,7 +107,7 @@ def display_members(ctx, mlist, role, regular, digest,
         if email_only or not address.display_name:
             print(address.original_email, file=outfp)
         else:
-            print(f'{address.display_name} <{address.original_email}>',
+            print(formataddr((address.display_name, address.original_email)),
                   file=outfp)
 
 
diff --git a/src/mailman/commands/cli_syncmembers.py b/src/mailman/commands/cli_syncmembers.py
index 9547ef98ce..3d732a5979 100644
--- a/src/mailman/commands/cli_syncmembers.py
+++ b/src/mailman/commands/cli_syncmembers.py
@@ -20,7 +20,8 @@
 import sys
 import click
 
-from email.utils import formataddr, parseaddr
+from email.utils import parseaddr
+from mailman.utilities.email import formataddr
 from mailman.app.membership import delete_member
 from mailman.core.i18n import _
 from mailman.database.transaction import transactional
diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py
index d3b90d2015..a6ee190852 100644
--- a/src/mailman/commands/eml_membership.py
+++ b/src/mailman/commands/eml_membership.py
@@ -18,7 +18,8 @@
 """The email commands 'join' and 'subscribe'."""
 
 from email.header import decode_header, make_header
-from email.utils import formataddr, parseaddr
+from email.utils import  parseaddr
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.address import InvalidEmailAddressError
 from mailman.interfaces.command import ContinueProcessing, IEmailCommand
diff --git a/src/mailman/commands/eml_who.py b/src/mailman/commands/eml_who.py
index e536eae6af..c99916399f 100644
--- a/src/mailman/commands/eml_who.py
+++ b/src/mailman/commands/eml_who.py
@@ -19,7 +19,7 @@
 
 import re
 
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.command import ContinueProcessing, IEmailCommand
 from mailman.interfaces.member import DeliveryMode, DeliveryStatus
diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py
index 4f1184c017..c6f909ac42 100644
--- a/src/mailman/core/switchboard.py
+++ b/src/mailman/core/switchboard.py
@@ -30,6 +30,7 @@ import email
 import pickle
 import hashlib
 import logging
+from email import policy
 
 from mailman.config import config
 from mailman.email.message import Message
@@ -159,7 +160,7 @@ class Switchboard:
             # have to generate the message later when we do size restriction
             # checking.
             original_size = len(msg)
-            msg = email.message_from_string(msg, Message)
+            msg = email.message_from_string(msg, Message, policy=policy.default)
             msg.original_size = original_size
             data['original_size'] = original_size
         return msg, data
diff --git a/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py b/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py
new file mode 100644
index 0000000000..e15c354a88
--- /dev/null
+++ b/src/mailman/database/alembic/versions/e46c5b3c876b_add_ascii_email_and_domain_name_columns.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2020-2022 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Add ascii email and domain name columns
+
+Revision ID: e46c5b3c876b
+Revises: 98224512c9c2
+Create Date: 2022-09-23 14:09:48.711761
+
+"""
+from alembic import op
+from sqlalchemy import Column, String
+
+# revision identifiers, used by Alembic.
+revision = 'e46c5b3c876b'
+down_revision = '98224512c9c2'
+
+
+def upgrade():
+    op.add_column('domain', Column('mail_host_alabel', String))
+    op.create_index(
+        op.f('domain_mail_host_alabel_key'), 'domain', ['mail_host_alabel'],
+        unique=True)
+    op.add_column('address', Column('email_alabel', String))
+    op.create_index(
+        op.f('ix_address_email_alabel'), 'address', ['mail_host_alabel'],
+        unique=True)
+    op.add_column('mailinglist', Column('mail_host_alabel', String))
+    op.add_column('ban', Column('email_alabel', String))
+    pass
+
+
+def downgrade():
+    op.drop_column('domain', Column('domain_host_alabel', String))
+    op.drop_column('address', Column('email_alabel', String))
+    op.drop_column('mailinglist', Column('mail_host_alabel', String))
+    op.drop_column('ban', Column('email_alabel', String))
+    pass
diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py
index 28a1a4632c..3ceba5243a 100644
--- a/src/mailman/database/types.py
+++ b/src/mailman/database/types.py
@@ -179,7 +179,7 @@ class SAUnicodeXL(TypeDecorator):
 @compiles(SAUnicodeXL, 'mysql')
 def compile_sa_unicode_xl(element, compiler, **kw):
     # We hardcode the collate here to make string comparison case sensitive.
-    return 'VARCHAR(20000) COLLATE utf8_bin'              # pragma: nocover
+    return 'VARCHAR(20000) COLLATE utf8_bin'  # pragma: nocover
 
 
 @compiles(SAUnicodeXL)
@@ -206,4 +206,4 @@ def default_sa_text(element, compiler, **kw):
 @compiles(SAText, 'mysql')
 def compile_sa_text(element, compiler, **kw):
     # We hardcode the collate here to make string comparison case sensitive.
-    return 'TEXT COLLATE utf8mb4_bin'                     # pragma: nocover
+    return 'TEXT COLLATE utf8mb4_bin'  # pragma: nocover
diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py
index 221fab4f9b..cbc57f7f9e 100644
--- a/src/mailman/email/message.py
+++ b/src/mailman/email/message.py
@@ -40,7 +40,7 @@ COMMASPACE = ', '
 
 
 @public
-class Message(email.message.Message):
+class Message(email.message.EmailMessage):
     # BAW: For debugging w/ bin/dumpdb.  Apparently pprint uses repr.
     def __repr__(self):
         return self.__str__()
@@ -52,9 +52,9 @@ class Message(email.message.Message):
         # Work around for https://bugs.python.org/issue27321 and
         # https://bugs.python.org/issue32330.
         try:
-            value = email.message.Message.as_string(self)
+            value = email.message.EmailMessage.as_string(self)
         except (KeyError, LookupError, UnicodeEncodeError):
-            value = email.message.Message.as_bytes(self).decode(
+            value = email.message.EmailMessage.as_bytes(self).decode(
                 'ascii', 'replace')
         # Also ensure no unicode surrogates in the returned string.
         return email.utils._sanitize(value)
@@ -62,9 +62,9 @@ class Message(email.message.Message):
     def as_bytes(self):
         # Workaround for https://bugs.python.org/issue41307.
         try:
-            value = email.message.Message.as_bytes(self)
+            value = email.message.EmailMessage.as_bytes(self)
         except UnicodeEncodeError:
-            value = email.message.Message.as_string(self).encode('utf-8')
+            value = email.message.EmailMessage.as_string(self).encode('utf-8')
         return value
 
     @property
diff --git a/src/mailman/email/tests/test_validate.py b/src/mailman/email/tests/test_validate.py
index cd1ffc152d..2fb0023228 100644
--- a/src/mailman/email/tests/test_validate.py
+++ b/src/mailman/email/tests/test_validate.py
@@ -26,6 +26,19 @@ from mailman.interfaces.address import (
 from mailman.testing.layers import ConfigLayer
 from zope.component import getUtility
 
+parametrized_international_email = [
+    ("食べる人@野菜と肉.com",),
+    ("çà@testé.dé",),
+    ("المديلمنتدب@نامه‌ای.com",),
+    ("dåkiØ@faß.de",)]
+
+parametrized_invalid_email = [
+    ("email.com",),
+    ("@email.com",),
+    ("invalid@email",),
+    ("invalid@",),
+    ("",)
+]
 
 class TestValidate(unittest.TestCase):
     """Test email validation."""
@@ -58,3 +71,14 @@ class TestValidate(unittest.TestCase):
         self.assertRaises(InvalidEmailAddressError,
                           self.validate,
                           'bad_address')
+
+    def test_valid_email(self):
+        assert_that(self.validator.is_valid('valid@email.com')).is_true()
+
+    @parameterized.expand(parametrized_international_email)
+    def test_valid_email_international(self, email):
+        assert_that(self.validator.is_valid(email)).is_true()
+
+    @parameterized.expand(parametrized_invalid_email)
+    def test_invalid_email(self, invalid_email):
+        assert_that(self.validator.is_valid(invalid_email)).is_false()
diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py
index a194a050ef..2ee1aa3333 100644
--- a/src/mailman/email/validate.py
+++ b/src/mailman/email/validate.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2023 by the Free Software Foundation, Inc.
+# Copyright (C) 2009-2022 by the Free Software Foundation, Inc.
 #
 # This file is part of GNU Mailman.
 #
@@ -17,7 +17,9 @@
 
 """Email address validation."""
 
-import re
+from email_validator import EmailNotValidError, validate_email
+from public import public
+from zope.interface import implementer
 
 from mailman.interfaces.address import (
     IEmailValidator,
@@ -46,20 +48,12 @@ class Validator:
         """See `IEmailValidator`."""
         if not email:
             return False
-        user, domain_parts = split_email(email)
-        # Strip quotes from quoted local.
-        user = re.sub('^"(.*)"$', r'\1', user)
-        if not user or len(_valid_local.sub('', user)) > 0:
-            return False
-        # Local, unqualified addresses are not allowed.
-        if not domain_parts:
-            return False
-        if len(domain_parts) < 2:
+        try:
+            # Validate the email address.
+            validate_email(email, check_deliverability=False)
+            return True
+        except Exception:
             return False
-        for p in domain_parts:
-            if len(p) == 0 or p[0] == '-' or len(_valid_domain.sub('', p)) > 0:
-                return False
-        return True
 
     def validate(self, email):
         """Validate an email address.
diff --git a/src/mailman/handlers/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py
index cf080291d8..833f2139d8 100644
--- a/src/mailman/handlers/avoid_duplicates.py
+++ b/src/mailman/handlers/avoid_duplicates.py
@@ -25,7 +25,8 @@ warning header, or pass it through, depending on the user's preferences.
 
 import re
 
-from email.utils import formataddr, getaddresses
+from email.utils import getaddresses
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.handler import IHandler
 from public import public
diff --git a/src/mailman/handlers/cleanse.py b/src/mailman/handlers/cleanse.py
index 7543643af1..49074108cd 100644
--- a/src/mailman/handlers/cleanse.py
+++ b/src/mailman/handlers/cleanse.py
@@ -20,7 +20,8 @@
 import re
 import logging
 
-from email.utils import formataddr, make_msgid
+from email.utils import make_msgid
+from mailman.utilities.email import formataddr
 from mailman.config import config
 from mailman.core.i18n import _
 from mailman.handlers.cook_headers import uheader
diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py
index 57c80d9be9..33eba01589 100644
--- a/src/mailman/handlers/cook_headers.py
+++ b/src/mailman/handlers/cook_headers.py
@@ -20,7 +20,8 @@
 import logging
 
 from email.header import Header
-from email.utils import formataddr, getaddresses, parseaddr
+from email.utils import  getaddresses, parseaddr
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.handler import IHandler
 from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py
index bc95760ba0..5603c6a56e 100644
--- a/src/mailman/handlers/decorate.py
+++ b/src/mailman/handlers/decorate.py
@@ -22,7 +22,7 @@ import copy
 import logging
 
 from email.mime.text import MIMEText
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from mailman.archiving.mailarchive import MailArchive
 from mailman.core.i18n import _
 from mailman.interfaces.handler import IHandler
diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py
index 95b1c9a3dc..0ec2b13db4 100644
--- a/src/mailman/handlers/dmarc.py
+++ b/src/mailman/handlers/dmarc.py
@@ -31,7 +31,8 @@ import logging
 from email.header import decode_header, Header
 from email.mime.message import MIMEMessage
 from email.mime.text import MIMEText
-from email.utils import formataddr, getaddresses, make_msgid
+from email.utils import  getaddresses, make_msgid
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.interfaces.handler import IHandler
 from mailman.interfaces.mailinglist import DMARCMitigateAction, ReplyToMunging
diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py
index bac9857cc0..d7229af7f5 100644
--- a/src/mailman/handlers/rfc_2369.py
+++ b/src/mailman/handlers/rfc_2369.py
@@ -19,7 +19,7 @@
 
 import logging
 
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from mailman.core.i18n import _
 from mailman.handlers.cook_headers import uheader
 from mailman.interfaces.archiver import ArchivePolicy
diff --git a/src/mailman/handlers/subject_prefix.py b/src/mailman/handlers/subject_prefix.py
index 577149cee4..94ca62fdb2 100644
--- a/src/mailman/handlers/subject_prefix.py
+++ b/src/mailman/handlers/subject_prefix.py
@@ -19,6 +19,8 @@
 
 import re
 
+import logging
+
 from contextlib import suppress
 from email.header import decode_header, Header, make_header
 from mailman.core.i18n import _
@@ -202,5 +204,5 @@ class SubjectPrefix:
                 mlist, msgdata, subject, prefix, prefix_pattern, ws)
             if new_subject is not None:
                 del msg['subject']
-                msg['Subject'] = new_subject
+                msg['subject'] = str(new_subject)
                 return
diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py
index d2952b4325..3e5516c6bb 100644
--- a/src/mailman/interfaces/address.py
+++ b/src/mailman/interfaces/address.py
@@ -101,6 +101,9 @@ class IAddress(Interface):
         case preserved email address; `email` will always be lower case.
         """)
 
+    email_ascii = Attribute(
+        """transformed ascii email address""")
+
     display_name = Attribute(
         """Optional display name associated with the email address.""")
 
diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py
index ac3f0962f9..664f8390e8 100644
--- a/src/mailman/model/address.py
+++ b/src/mailman/model/address.py
@@ -16,8 +16,13 @@
 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
 
 """Model for addresses."""
+from public import public
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
+from sqlalchemy.orm import backref, relationship
+from zope.component import getUtility
+from zope.event import notify
+from zope.interface import implementer
 
-from email.utils import formataddr
 from mailman.database.model import Model
 from mailman.database.types import SAUnicode, SAUnicode4Byte
 from mailman.interfaces.address import (
@@ -26,12 +31,9 @@ from mailman.interfaces.address import (
     IEmailValidator,
 )
 from mailman.utilities.datetime import now
-from public import public
-from sqlalchemy import Column, DateTime, ForeignKey, Integer
-from sqlalchemy.orm import backref, relationship
-from zope.component import getUtility
-from zope.event import notify
-from zope.interface import implementer
+from mailman.utilities.email import formataddr
+from mailman.utilities.ua_utils import (transform_email_domain_to_ascii,
+                                        transform_email_to_utf8)
 
 
 @public
@@ -43,6 +45,7 @@ class Address(Model):
 
     id = Column(Integer, primary_key=True)
     email = Column(SAUnicode, index=True, unique=True)
+    email_alabel = Column(String, index=True, unique=True)
     _original = Column(SAUnicode)
     display_name = Column(SAUnicode4Byte)
     _verified_on = Column('verified_on', DateTime)
@@ -59,6 +62,8 @@ class Address(Model):
         getUtility(IEmailValidator).validate(email)
         lower_case = email.lower()
         self.email = lower_case
+        self.email_alabel = transform_email_domain_to_ascii(self.email)
+        self.email = transform_email_to_utf8(self.email_alabel)
         self.display_name = display_name
         self._original = (None if lower_case == email else email)
         self.registered_on = now()
diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py
index b8d8dfc58f..4360970472 100644
--- a/src/mailman/model/bans.py
+++ b/src/mailman/model/bans.py
@@ -25,9 +25,11 @@ from mailman.database.types import SAUnicode
 from mailman.interfaces.bans import IBan, IBanManager
 from mailman.utilities.queries import QuerySequence
 from public import public
-from sqlalchemy import Column, Integer, select
+from sqlalchemy import Column, Integer, select, String, or_
 from zope.interface import implementer
 
+from mailman.utilities.ua_utils import transform_email_domain_to_ascii
+
 
 @public
 @implementer(IBan)
@@ -38,11 +40,13 @@ class Ban(Model):
 
     id = Column(Integer, primary_key=True)
     email = Column(SAUnicode, index=True)
+    email_alabel = Column(String, index=True)
     list_id = Column(SAUnicode, index=True)
 
     def __init__(self, email, list_id):
         super().__init__()
         self.email = email
+        self.email_alabel = transform_email_domain_to_ascii(email)
         self.list_id = list_id
 
 
@@ -58,7 +62,9 @@ class BanManager:
     @dbconnection
     def ban(self, store, email):
         """See `IBanManager`."""
-        bans = store.query(Ban).filter_by(email=email, list_id=self._list_id)
+        bans = store.query(Ban).filter(or_(Ban.email == email,
+                                           Ban.email_alabel == email),
+                                       Ban.list_id == self._list_id)
         if bans.count() == 0:
             ban = Ban(email, self._list_id)
             store.add(ban)
@@ -66,8 +72,9 @@ class BanManager:
     @dbconnection
     def unban(self, store, email):
         """See `IBanManager`."""
-        ban = store.query(Ban).filter_by(
-            email=email, list_id=self._list_id).first()
+        ban = store.query(Ban).filter(or_(Ban.email == email,
+                                          Ban.email_alabel == email),
+                                      Ban.list_id == self._list_id).first()
         if ban is not None:
             store.delete(ban)
 
@@ -78,7 +85,9 @@ class BanManager:
         if list_id is None:
             # The client is asking for global bans.  Look up bans on the
             # specific email address first.
-            bans = store.query(Ban).filter_by(email=email, list_id=None)
+            bans = store.query(Ban).filter(or_(Ban.email == email,
+                                               Ban.email_alabel == email),
+                                           Ban.list_id == None)
             if bans.count() > 0:
                 return True
             # And now look for global pattern bans.
@@ -89,11 +98,15 @@ class BanManager:
                     return True
         else:
             # This is a list-specific ban.
-            bans = store.query(Ban).filter_by(email=email, list_id=list_id)
+            bans = store.query(Ban).filter(or_(Ban.email == email,
+                                               Ban.email_alabel == email),
+                                           Ban.list_id == list_id)
             if bans.count() > 0:
                 return True
             # Try global bans next.
-            bans = store.query(Ban).filter_by(email=email, list_id=None)
+            bans = store.query(Ban).filter(or_(Ban.email == email,
+                                               Ban.email_alabel == email),
+                                           Ban.list_id == None)
             if bans.count() > 0:
                 return True
             # Now try specific mailing list bans, but with a pattern.
diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py
index c64d648e30..d2067d858d 100644
--- a/src/mailman/model/domain.py
+++ b/src/mailman/model/domain.py
@@ -33,12 +33,15 @@ from mailman.interfaces.user import IUser
 from mailman.interfaces.usermanager import IUserManager
 from mailman.model.mailinglist import MailingList
 from public import public
-from sqlalchemy import Column, func, Integer, select
+from sqlalchemy import Column, func, Integer, select, String, or_
 from sqlalchemy.orm import relationship
 from zope.component import getUtility
 from zope.event import notify
 from zope.interface import implementer
 
+from mailman.utilities.ua_utils import (convert_to_alabel,
+                                        validate_and_transform_to_utf8)
+
 
 @public
 @implementer(IDomain)
@@ -50,6 +53,7 @@ class Domain(Model):
     id = Column(Integer, primary_key=True)
 
     mail_host = Column(SAUnicode, unique=True)
+    mail_host_alabel = Column(String, unique=True)
     description = Column(SAUnicode4Byte)
     owners = relationship('User',
                           secondary='domain_owner',
@@ -59,7 +63,8 @@ class Domain(Model):
     def __init__(self, mail_host,
                  description=None,
                  owners=None,
-                 alias_domain=None):
+                 alias_domain=None,
+                 idna_domain=None):
         """Create and register a domain.
 
         :param mail_host: The host name for the email interface.
@@ -71,7 +76,8 @@ class Domain(Model):
         :param alias_domain: Alternate domain for Postfix
         :type alias_domain: string
         """
-        self.mail_host = mail_host
+        self.mail_host = validate_and_transform_to_utf8(mail_host)
+        self.mail_host_alabel = convert_to_alabel(mail_host)
         self.description = description
         if owners is not None:
             self.add_owners(owners)
@@ -83,7 +89,7 @@ class Domain(Model):
         """See `IDomain`."""
         yield from store.query(MailingList).filter(
             MailingList.mail_host == self.mail_host
-            ).order_by(MailingList._list_id)
+        ).order_by(MailingList._list_id)
 
     def __repr__(self):
         """repr(a_domain)"""
@@ -158,7 +164,9 @@ class DomainManager:
     @dbconnection
     def get(self, store, mail_host, default=None):
         """See `IDomainManager`."""
-        domains = store.query(Domain).filter_by(mail_host=mail_host)
+        domains = store.query(Domain).filter(or_(
+            Domain.mail_host == mail_host,
+            Domain.mail_host_alabel == mail_host))
         if domains.count() < 1:
             return default
         assert domains.count() == 1, (
diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py
index 09b3297f63..477f96af73 100644
--- a/src/mailman/model/listmanager.py
+++ b/src/mailman/model/listmanager.py
@@ -43,6 +43,8 @@ from sqlalchemy import select
 from zope.event import notify
 from zope.interface import implementer
 
+from mailman.utilities.ua_utils import validate_and_transform_to_utf8
+
 
 @public
 @implementer(IListManager)
@@ -53,9 +55,11 @@ class ListManager:
     def create(self, store, fqdn_listname):
         """See `IListManager`."""
         fqdn_listname = fqdn_listname.lower()
-        listname, at, hostname = fqdn_listname.partition('@')
+        listname, hostname = fqdn_listname.rsplit('@', 1)
+        listname = listname.strip("'")
         if len(hostname) == 0:
             raise InvalidEmailAddressError(fqdn_listname)
+        hostname = validate_and_transform_to_utf8(hostname)
         list_id = '{}.{}'.format(listname, hostname)
         notify(ListCreatingEvent(fqdn_listname))
         mlist = store.query(MailingList).filter_by(_list_id=list_id).first()
diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py
index f54a6b6cad..d4fe042526 100644
--- a/src/mailman/model/mailinglist.py
+++ b/src/mailman/model/mailinglist.py
@@ -81,6 +81,7 @@ from sqlalchemy import (
     LargeBinary,
     PickleType,
     select,
+    String,
 )
 from sqlalchemy.event import listen
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -91,6 +92,7 @@ from zope.component import getUtility
 from zope.event import notify
 from zope.interface import implementer
 
+from mailman.utilities.ua_utils import convert_to_alabel
 
 SPACE = ' '
 UNDERSCORE = '_'
@@ -111,6 +113,7 @@ class MailingList(Model):
     # List identity
     list_name = Column(SAUnicode, index=True)
     mail_host = Column(SAUnicode, index=True)
+    mail_host_alabel = Column(String, index=True)
     _list_id = Column('list_id', SAUnicode, index=True, unique=True)
     allow_list_posts = Column(Boolean)
     include_rfc2369_headers = Column(Boolean)
@@ -230,6 +233,7 @@ class MailingList(Model):
         assert hostname, 'Bad list name: {0}'.format(fqdn_listname)
         self.list_name = listname
         self.mail_host = hostname
+        self.mail_host_alabel = convert_to_alabel(self.mail_host)
         self._list_id = '{0}.{1}'.format(listname, hostname)
         # For the pending database
         self.next_request_id = 1
diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py
index f7c4c54c72..984c228d86 100644
--- a/src/mailman/model/roster.py
+++ b/src/mailman/model/roster.py
@@ -104,14 +104,16 @@ class AbstractRoster:
         members_a = store.query(Member).filter(
             Member.list_id == self._mlist.list_id,
             Member.role == self.role,
-            Address.email == email,
+            or_(Address.email == email,
+                Address.email_alabel == email),
             Member.address_id == Address.id)
         # Here's a query that finds all members subscribed with their
         # preferred address.
         members_u = store.query(Member).filter(
             Member.list_id == self._mlist.list_id,
             Member.role == self.role,
-            Address.email == email,
+            or_(Address.email == email,
+                Address.email_alabel == email),
             Member.user_id == User.id,
             User._preferred_address_id == Address.id)
         return members_a.union(members_u).all()
diff --git a/src/mailman/model/tests/test_domain.py b/src/mailman/model/tests/test_domain.py
index a6afdec389..79e6f69967 100644
--- a/src/mailman/model/tests/test_domain.py
+++ b/src/mailman/model/tests/test_domain.py
@@ -181,6 +181,16 @@ class TestDomainManager(unittest.TestCase):
         self.assertEqual([owner.addresses[0].email for owner in domain.owners],
                          ['anne@example.org', 'bart@example.net'])
 
+    def test_add_domain_utf8(self):
+        domain = self._manager.add('例題.com')
+        self.assertEqual(domain.mail_host_alabel, 'xn--fsqr78o.com')
+        self.assertEqual(domain.mail_host, '例題.com')
+
+    def test_add_domain_alabel(self):
+        domain = self._manager.add('xn--fsqr78o.com')
+        self.assertEqual(domain.mail_host, '例題.com')
+        self.assertEqual(domain.mail_host_alabel, 'xn--fsqr78o.com')
+
 
 class TestDomain(unittest.TestCase):
     layer = ConfigLayer
diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py
index 6d5fac2f58..a8b46f87c3 100644
--- a/src/mailman/model/usermanager.py
+++ b/src/mailman/model/usermanager.py
@@ -16,7 +16,6 @@
 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
 
 """A user manager."""
-
 from mailman.database.transaction import dbconnection
 from mailman.interfaces.address import ExistingAddressError
 from mailman.interfaces.usermanager import IUserManager
@@ -31,7 +30,6 @@ from public import public
 from sqlalchemy import or_, select
 from zope.interface import implementer
 
-
 @public
 @implementer(IUserManager)
 class UserManager:
@@ -108,7 +106,9 @@ class UserManager:
     @dbconnection
     def create_address(self, store, email, display_name=None):
         """See `IUserManager`."""
-        addresses = store.query(Address).filter(Address.email == email.lower())
+        addresses = store.query(Address).filter(or_(
+                            Address.email == email.lower(),
+                            Address.email_alabel == email.lower()))
         if addresses.count() == 1:
             found = addresses[0]
             raise ExistingAddressError(found.original_email)
@@ -142,8 +142,9 @@ class UserManager:
     @dbconnection
     def get_address(self, store, email):
         """See `IUserManager`."""
-        return store.query(
-            Address).filter_by(email=email.lower()).one_or_none()
+        return store.query(Address).filter(
+            or_(Address.email == email.lower(),
+                Address.email_alabel == email.lower())).one_or_none()
 
     @property
     @dbconnection
@@ -170,6 +171,7 @@ class UserManager:
         q = '%{}%'.format(query)
         yield from store.query(User).join(
             Address, Address.user_id == User.id).filter(
-                or_(User.display_name.ilike(q),
-                    Address.display_name.ilike(q),
-                    Address.email.ilike(q)))
+            or_(User.display_name.ilike(q),
+                Address.display_name.ilike(q),
+                Address.email.ilike(q),
+                Address.email_alabel == q))
diff --git a/src/mailman/mta/connection.py b/src/mailman/mta/connection.py
index cd4759a069..25c90635d1 100644
--- a/src/mailman/mta/connection.py
+++ b/src/mailman/mta/connection.py
@@ -24,12 +24,14 @@ import logging
 import smtplib
 
 from contextlib import suppress
-from email.message import Message
+from email.message import EmailMessage
 from lazr.config import as_boolean
 from mailman.config import config
 from mailman.interfaces.configuration import InvalidConfigurationError
 from public import public
 
+from mailman.utilities.email import split_email
+from mailman.utilities.ua_utils import transform_email_domain_to_ascii
 
 log = logging.getLogger('mailman.smtp')
 
@@ -58,6 +60,7 @@ def as_SecureMode(s):
 
 class Connection:
     """Manage a connection to the SMTP server."""
+
     def __init__(self, host, port, sessions_per_connection,
                  smtp_user=None, smtp_pass=None,
                  secure_mode=SecureMode.INSECURE,
@@ -135,16 +138,17 @@ class Connection:
         # object is preferred because of the treatment of line endings.
         if isinstance(msg, str):
             msg = msg.encode('ascii', 'replace')
-        assert isinstance(msg, bytes) or isinstance(msg, Message), \
+        assert isinstance(msg, bytes) or isinstance(msg, EmailMessage), \
             'Connection.sendmail received an invalid msg arg.'
         try:
-            log.debug('envsender: %s, recipients: %s, size(msg): %s',
-                      envsender, recipients, len(msg))
-            if isinstance(msg, Message):
-                results = self._connection.send_message(msg, envsender,
-                                                        recipients)
-            else:
-                results = self._connection.sendmail(envsender, recipients, msg)
+            results = self._sendmail(envsender, recipients, msg)
+        except smtplib.SMTPNotSupportedError as e:
+            recipients_ascii = list(
+                filter(lambda recipient: recipient is not None,
+                       map(self._smtputf8_fallback, recipients)))
+            if len(recipients_ascii) != len(recipients):
+                raise e
+            results = self.sendmail(envsender, recipients, msg)
         except smtplib.SMTPException:
             # For safety, close this connection.  The next send attempt will
             # automatically re-open it.  Pass the exception on up.
@@ -159,6 +163,24 @@ class Connection:
             self.quit()
         return results
 
+    def _smtputf8_fallback(self, recipient):
+        # first validate that the local_part is ascii,
+        # punycode being applied only to domain names
+        # encoding local part may result in errors.
+        local_part, domain = split_email(recipient)
+        if not local_part.isascii():
+            return
+        return transform_email_domain_to_ascii(recipient)
+
+    def _sendmail(self, envsender, recipients, msg):
+        log.debug('envsender: %s, recipients: %s, size(msg): %s',
+                  envsender, recipients, len(msg))
+        if isinstance(msg, EmailMessage):
+            return self._connection.send_message(msg, envsender,
+                                                 recipients)
+        else:
+            return self._connection.sendmail(envsender, recipients, msg)
+
     def quit(self):
         """Mimic `smtplib.SMTP.quit`."""
         if self._connection is None:
diff --git a/src/mailman/mta/personalized.py b/src/mailman/mta/personalized.py
index 0dfebef460..a477b9b558 100644
--- a/src/mailman/mta/personalized.py
+++ b/src/mailman/mta/personalized.py
@@ -18,7 +18,7 @@
 """Personalized delivery."""
 
 from email.header import Header
-from email.utils import formataddr
+from mailman.utilities.email import formataddr
 from mailman.interfaces.mailinglist import Personalization
 from mailman.interfaces.usermanager import IUserManager
 from mailman.mta.verp import VERPDelivery
diff --git a/src/mailman/mta/tests/test_connection.py b/src/mailman/mta/tests/test_connection.py
index b0aa510944..7c1d9b1002 100644
--- a/src/mailman/mta/tests/test_connection.py
+++ b/src/mailman/mta/tests/test_connection.py
@@ -18,6 +18,8 @@
 """Test MTA connections."""
 
 import unittest
+from smtplib import SMTPAuthenticationError, SMTPNotSupportedError
+from unittest.mock import patch
 
 from mailman.config import config
 from mailman.mta.connection import Connection, SecureMode
@@ -26,8 +28,6 @@ from mailman.testing.helpers import (
     specialized_message_from_string as mfs,
 )
 from mailman.testing.layers import SMTPLayer, SMTPSLayer, STARTTLSLayer
-from smtplib import SMTPAuthenticationError, SMTPNotSupportedError
-from unittest.mock import patch
 
 
 class TestConnection(unittest.TestCase):
@@ -156,7 +156,6 @@ Subject: aardvarks
 
 
 class TestSMTPConnectionCount(_ConnectionCounter, unittest.TestCase):
-
     layer = SMTPLayer
 
     def setUp(self):
@@ -174,7 +173,6 @@ class TestSMTPConnectionCount(_ConnectionCounter, unittest.TestCase):
 
 
 class TestSMTPSConnectionCount(_ConnectionCounter, unittest.TestCase):
-
     layer = SMTPSLayer
 
     def setUp(self):
@@ -182,11 +180,10 @@ class TestSMTPSConnectionCount(_ConnectionCounter, unittest.TestCase):
         self.connection = Connection(
             config.mta.smtp_host, int(config.mta.smtp_port), 0,
             secure_mode=SecureMode.IMPLICIT, verify_cert=False
-            )
+        )
 
 
 class TestSTARTTLSConnectionCount(_ConnectionCounter, unittest.TestCase):
-
     layer = STARTTLSLayer
 
     def setUp(self):
@@ -194,7 +191,7 @@ class TestSTARTTLSConnectionCount(_ConnectionCounter, unittest.TestCase):
         self.connection = Connection(
             config.mta.smtp_host, int(config.mta.smtp_port), 0,
             secure_mode=SecureMode.STARTTLS, verify_cert=False
-            )
+        )
 
 
 class TestSTARTTLSNotSupported(unittest.TestCase):
@@ -204,7 +201,7 @@ class TestSTARTTLSNotSupported(unittest.TestCase):
         connection = Connection(
             config.mta.smtp_host, int(config.mta.smtp_port), 0,
             secure_mode=SecureMode.STARTTLS, verify_cert=False
-            )
+        )
         msg_text = """\
 From: anne@example.com
 To: bart@example.com
diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py
index 15a8026b60..f6ad19a674 100644
--- a/src/mailman/rest/addresses.py
+++ b/src/mailman/rest/addresses.py
@@ -55,6 +55,7 @@ class _AddressBase(CollectionMixin):
         # email address.
         representation = dict(
             email=address.email,
+            email_alabel=address.email_alabel,
             original_email=address.original_email,
             registered_on=address.registered_on,
             self_link=self.api.path_to('addresses/{}'.format(address.email)),
diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py
index d592ad62cb..38a6d1e168 100644
--- a/src/mailman/rest/lists.py
+++ b/src/mailman/rest/lists.py
@@ -16,7 +16,6 @@
 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
 
 """REST for mailing lists."""
-
 from lazr.config import as_boolean
 from mailman.app.digests import (
     bump_digest_number_and_volume,
@@ -65,7 +64,6 @@ from mailman.rest.validator import (
 from public import public
 from zope.component import getUtility
 
-
 def member_matcher(segments):
     """A matcher of member URLs inside mailing lists.
 
diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py
index 1a82cef3cc..07a7d80508 100644
--- a/src/mailman/rest/validator.py
+++ b/src/mailman/rest/validator.py
@@ -16,7 +16,6 @@
 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
 
 """REST web form validation."""
-
 import re
 
 from lazr.config import as_boolean
diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py
index 8927f79b91..277df4e88b 100644
--- a/src/mailman/runners/digest.py
+++ b/src/mailman/runners/digest.py
@@ -60,7 +60,7 @@ class Digester:
         self._message = self._make_message()
         self._digest_part = self._make_digest_part()
         self._message['From'] = mlist.request_address
-        self._message['Subject'] = self._subject
+        self._message['Subject'] = str(self._subject)
         self._message['To'] = mlist.posting_address
         self._message['Reply-To'] = mlist.posting_address
         self._message['Date'] = formatdate(localtime=True)
diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py
index 804c79a141..d36def9127 100644
--- a/src/mailman/runners/lmtp.py
+++ b/src/mailman/runners/lmtp.py
@@ -37,6 +37,7 @@ so that the peer mail server can provide better diagnostics.
 import re
 import email
 import logging
+from email import policy
 
 from aiosmtpd.controller import Controller
 from aiosmtpd.lmtp import LMTP
@@ -182,7 +183,9 @@ class LMTPHandler:
             listnames = set(getUtility(IListManager).names)
             # Parse the message data.  If there are any defects in the
             # message, reject it right away; it's probably spam.
-            msg = email.message_from_bytes(envelope.content, Message)
+            msg = email.message_from_bytes(envelope.content,
+                                           Message,
+                                           policy=policy.default)
             msg.set_unixfrom(envelope.mail_from)
         except Exception:
             elog.exception('LMTP message parsing')
diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py
index 87f591f29a..83e77c7c09 100644
--- a/src/mailman/testing/helpers.py
+++ b/src/mailman/testing/helpers.py
@@ -30,7 +30,7 @@ import datetime
 import threading
 
 from contextlib import contextmanager, suppress
-from email import message_from_bytes, message_from_string
+from email import message_from_bytes, message_from_string, policy
 from lazr.config import as_timedelta
 from mailman.bin.master import Loop as Master
 from mailman.config import config
@@ -255,7 +255,7 @@ def get_nntp_server(cleanups):
     class NNTPProxy:                                              # noqa: E306
         def get_message(self):
             args = nntpd.post.call_args
-            return message_from_bytes(args[0][0].read())
+            return message_from_bytes(args[0][0].read(), policy=policy.default)
     return NNTPProxy()
 
 
@@ -524,7 +524,7 @@ def specialized_message_from_string(unicode_text):
     # This mimic what Switchboard.dequeue() does when parsing a message from
     # text into a Message instance.
     original_size = len(unicode_text)
-    message = message_from_string(unicode_text, Message)
+    message = message_from_string(unicode_text, Message, policy=policy.default)
     message.original_size = original_size
     return message
 
diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py
index af6b7ccdab..c90acc94bf 100644
--- a/src/mailman/utilities/email.py
+++ b/src/mailman/utilities/email.py
@@ -16,11 +16,16 @@
 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
 
 """Email helpers."""
-
+import re
 from base64 import b32encode
+from email import utils
 from hashlib import sha1
+
 from public import public
 
+specialsre = re.compile(r'[][\\()<>@,:;".]')
+escapesre = re.compile(r'[\\"]')
+
 
 @public
 def split_email(address):
@@ -75,3 +80,25 @@ def add_message_hash(msg):
     del msg['x-message-id-hash']
     msg['X-Message-ID-Hash'] = hash32
     return hash32
+
+
+@public
+def formataddr(pair, charset='utf-8'):
+    name, address = pair
+    if address.isascii():
+        return utils.formataddr(pair)
+    else:
+        return formataddr_utf8(pair)
+
+
+def formataddr_utf8(pair, charset='utf-8'):
+    name, address = pair
+    if name:
+        return "%s <%s>" % (name, address)
+    else:
+        quotes = ''
+        if specialsre.search(name):
+            quotes = '"'
+        name = escapesre.sub(r'\\\g<0>', name)
+        return '%s%s%s <%s>' % (quotes, name, quotes, address)
+    return address
diff --git a/src/mailman/utilities/tests/test_email.py b/src/mailman/utilities/tests/test_email.py
index 798cdbc71a..c4ac86874f 100644
--- a/src/mailman/utilities/tests/test_email.py
+++ b/src/mailman/utilities/tests/test_email.py
@@ -19,11 +19,15 @@
 
 import unittest
 
+from assertpy import assert_that
+from parameterized import parameterized
+from zope.component import getUtility
+
+from mailman.email.tests.test_validate import parametrized_international_email
 from mailman.interfaces.messages import IMessageStore
 from mailman.testing.helpers import specialized_message_from_string as mfs
 from mailman.testing.layers import ConfigLayer
-from mailman.utilities.email import add_message_hash, split_email
-from zope.component import getUtility
+from mailman.utilities.email import add_message_hash, split_email, formataddr
 
 
 class TestEmail(unittest.TestCase):
@@ -154,3 +158,24 @@ X-Message-ID-Hash: abc
         self.assertEqual(len(x_message_id_hashes), 1)
         self.assertEqual(x_message_id_hashes[0],
                          'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
+
+
+class TestFormatAddr(unittest.TestCase):
+
+    def test_formataddr_when_email_ascii(self):
+        address = "test@example.com"
+        name = "test tester"
+        result = formataddr((name, address))
+        assert_that(result).is_equal_to("test tester <test@example.com>")
+
+    @parameterized.expand(parametrized_international_email)
+    def test_formataddr_on_valid_utf8_email_ascii_username(self, email):
+        name = 'test tester'
+        result = formataddr((name, email))
+        assert_that(result).is_equal_to("test tester <{}>".format(email))
+
+    @parameterized.expand(parametrized_international_email)
+    def test_formataddr_on_valid_email_utf8_username(self, email):
+        name = 'しげる タナカ'
+        result = formataddr((name, email))
+        assert_that(result).is_equal_to("しげる タナカ <{}>".format(email))
diff --git a/src/mailman/utilities/tests/test_ua_utils.py b/src/mailman/utilities/tests/test_ua_utils.py
new file mode 100644
index 0000000000..8186762240
--- /dev/null
+++ b/src/mailman/utilities/tests/test_ua_utils.py
@@ -0,0 +1,55 @@
+import unittest
+
+import idna
+from assertpy import assert_that
+from parameterized import parameterized
+
+from mailman.utilities.ua_utils import (convert_to_alabel,
+                                        validate_and_transform_to_utf8,
+                                        transform_email_to_utf8,
+                                        transform_email_domain_to_ascii)
+
+parametrized_domain_name = [
+    ("野菜と肉.com", "xn--o9jw62v5cc81t.com", "野菜と肉.com"),
+    ("testé.dé", "xn--test-epa.xn--d-bga", "testé.dé"),
+    ("المديلمنتدب.com", "xn--mgbcftb7jbgck5b.com", "المديلمنتدب.com"),
+    ("faß.de", "xn--fa-hia.de", "faß.de"),
+    ("例題.イオ", "xn--fsqr78o.xn--eckm", "例題.イオ"),
+    ("例題.xn--eckm", "xn--fsqr78o.xn--eckm", "例題.イオ")
+]
+
+parametrized_international_email = [
+    ("食べる人@野菜と肉.com", "食べる人@xn--o9jw62v5cc81t.com"),
+    ("çà@testé.dé", "çà@xn--test-epa.xn--d-bga"),
+    ("المديلمنتدب@نامه‌ای.com", "المديلمنتدب@xn--mgba3gch31f060k.com"),
+    ("dåkiØ@faß.de", "dåkiØ@xn--fa-hia.de")]
+
+
+class TestDomainTranslate(unittest.TestCase):
+
+    @parameterized.expand(parametrized_domain_name)
+    def test_normal_utf8_string(self, domain, aLabel, domain_utf8):
+        result = convert_to_alabel(domain)
+        assert_that(result).is_equal_to(aLabel)
+
+    def test_too_long_domain_name(self):
+        long_domain_name = "x" * 64
+        long_domain_name += ".com"
+        assert_that(convert_to_alabel).raises(idna.IDNAError).when_called_with(
+            long_domain_name)
+
+    def test_too_short_domain_name(self):
+        assert_that(convert_to_alabel).raises(idna.IDNAError).when_called_with(
+            '')
+
+    @parameterized.expand(parametrized_domain_name)
+    def test_convert_to_utf8(self, domain, alabel, domain_utf8):
+        assert_that(validate_and_transform_to_utf8(alabel)).is_equal_to(domain_utf8)
+
+    @parameterized.expand(parametrized_international_email)
+    def test_transform_to_ascii(self, email, expected):
+        assert_that(transform_email_domain_to_ascii(email)).is_equal_to(expected)
+
+    @parameterized.expand(parametrized_international_email)
+    def test_transform_email_to_ascii(self, expected, email):
+        assert_that(transform_email_to_utf8(email)).is_equal_to(expected)
diff --git a/src/mailman/utilities/ua_utils.py b/src/mailman/utilities/ua_utils.py
new file mode 100644
index 0000000000..8f764d1aae
--- /dev/null
+++ b/src/mailman/utilities/ua_utils.py
@@ -0,0 +1,49 @@
+import logging
+
+import idna
+
+from mailman.utilities.email import split_email
+
+logger = logging.getLogger("mailman.core")
+
+
+def convert_to_alabel(domain):
+    """
+    Convert a domain name from U-Label to A-Label
+    :param domain: The domain name
+    :return the conversion result
+    """
+    try:
+        # Encode the domain name in A-Label to be used by DNS, etc. client.
+        # IDNA package is fully IDNA 2008 compliant
+        domain_a_label = idna.encode(domain, uts46=True).decode('ascii')
+        logger.info(
+            f"Domain '{domain}' converted in A-Label is '{domain_a_label}'")
+        return domain_a_label
+    except idna.IDNAError as e:
+        # The label is invalid as per IDNA 2008
+        logger.error(f"Domain '{domain}' is invalid: {e}")
+        raise e
+    except Exception as e:
+        # Unexpected exception
+        logger.error(f"ERROR: {e}")
+        # as this does not seem to be an IDNA related exception, always raise
+        raise e
+
+
+def transform_email_domain_to_ascii(email):
+    local_part, domain = split_email(email)
+    domain_alabel = convert_to_alabel(".".join(domain))
+    return '{}@{}'.format(local_part, domain_alabel)
+
+
+def transform_email_to_utf8(email):
+    local_part, domain = split_email(email)
+    domain_utf8 = validate_and_transform_to_utf8(".".join(domain))
+    return '{}@{}'.format(local_part, domain_utf8)
+
+
+def validate_and_transform_to_utf8(domain):
+    if 'xn--' in domain:
+        return idna.decode(domain)
+    return domain
diff --git a/tox.ini b/tox.ini
index a453d5fd7c..ab9656dab6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -22,6 +22,8 @@ deps =
      pg: psycopg2-binary
      mysql: pymysql
      diffcov: diff_cover>=6.0
+     email_validator
+     parameterized==0.8.1
 passenv =
     MAILMAN_*
     PYTHON*
@@ -84,4 +86,4 @@ ignore =
     port_me/*
     *.yml
 ignore-bad-ideas =
-    src/mailman/testing/*
\ No newline at end of file
+    src/mailman/testing/*
-- 
GitLab


From 3023a6a40eed8469a4c4efadb9f5f0895c293ef4 Mon Sep 17 00:00:00 2001
From: Michel Bernier <mbernier@cofomo.com>
Date: Fri, 5 May 2023 16:28:54 -0400
Subject: [PATCH 2/3] Change lmtp communication for using utf8. Change
 connection with smtp to transform every email in punycode upon configuration.
 Change mailbox to use EAI.

---
 src/mailman/app/tests/test_bounces.py  | 12 +++++++
 src/mailman/config/schema.cfg          |  1 +
 src/mailman/model/tests/test_bans.py   |  2 +-
 src/mailman/mta/connection.py          |  2 ++
 src/mailman/runners/lmtp.py            |  2 +-
 src/mailman/runners/tests/test_lmtp.py | 17 +++++++++-
 src/mailman/utilities/mailbox.py       | 45 +++++++++++++++++++++++++-
 tox.ini                                |  3 +-
 8 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/src/mailman/app/tests/test_bounces.py b/src/mailman/app/tests/test_bounces.py
index a48f84dc8a..906a5e01c6 100644
--- a/src/mailman/app/tests/test_bounces.py
+++ b/src/mailman/app/tests/test_bounces.py
@@ -23,6 +23,8 @@ import shutil
 import tempfile
 import unittest
 
+from parameterized import parameterized
+
 from mailman.app.bounces import (
     bounce_message,
     maybe_forward,
@@ -46,6 +48,8 @@ from mailman.testing.helpers import (
 from mailman.testing.layers import ConfigLayer
 from zope.component import getUtility
 
+from mailman.utilities.tests.test_ua_utils import parametrized_international_email
+
 
 class TestVERP(unittest.TestCase):
     """Test header VERP detection."""
@@ -561,3 +565,11 @@ Subject: Ignore
         bounce_message(self._mlist, self._msg)
         # Nothing in the virgin queue means nothing's been bounced.
         get_queue_messages('virgin', expected_count=0)
+
+    @parameterized.expand(parametrized_international_email)
+    def test_internationalized_sender(self, email, email_transformed):
+        self._msg['from'] = email
+        bounce_message(self._mlist, self._msg)
+        # Nothing in the virgin queue means nothing's been bounced.
+        get_queue_messages('virgin', expected_count=0)
+
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index ae0f5c70a9..4ab14abf56 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -777,6 +777,7 @@ smtp_host: localhost
 smtp_port: 25
 smtp_user:
 smtp_pass:
+always_punycode: no
 
 # One of smtp/smtps/starttls, specifies the protocol Mailman will use when
 # connecting.  Typically will correspond to smtp_port: 25 -> smtp, 465 -> smtps,
diff --git a/src/mailman/model/tests/test_bans.py b/src/mailman/model/tests/test_bans.py
index a53125177b..e3e2462cf7 100644
--- a/src/mailman/model/tests/test_bans.py
+++ b/src/mailman/model/tests/test_bans.py
@@ -62,4 +62,4 @@ class TestMailingListBans(unittest.TestCase):
         # The results can be indexed.
         self.assertEqual(
             [self._manager.bans[i].email for i in range(count)],
-            ['ant@example.com', 'bee@example.com', 'cat@example.com'])
+            ['ant@example.com', 'bee@example.com', 'cat@example.com'])
\ No newline at end of file
diff --git a/src/mailman/mta/connection.py b/src/mailman/mta/connection.py
index 25c90635d1..366477824f 100644
--- a/src/mailman/mta/connection.py
+++ b/src/mailman/mta/connection.py
@@ -141,6 +141,8 @@ class Connection:
         assert isinstance(msg, bytes) or isinstance(msg, EmailMessage), \
             'Connection.sendmail received an invalid msg arg.'
         try:
+            if as_boolean(config.mta.always_punycode):
+               recipients = [transform_email_domain_to_ascii(recipient) for recipient in recipients]
             results = self._sendmail(envsender, recipients, msg)
         except smtplib.SMTPNotSupportedError as e:
             recipients_ascii = list(
diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py
index d36def9127..a9425269fe 100644
--- a/src/mailman/runners/lmtp.py
+++ b/src/mailman/runners/lmtp.py
@@ -283,7 +283,7 @@ class LMTPHandler:
 
 class LMTPController(Controller):
     def factory(self):
-        server = LMTP(self.handler)
+        server = LMTP(self.handler, enable_SMTPUTF8=True)
         server.__ident__ = 'GNU Mailman LMTP runner 2.0'
         return server
 
diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py
index 73ab1e749c..5ee3020346 100644
--- a/src/mailman/runners/tests/test_lmtp.py
+++ b/src/mailman/runners/tests/test_lmtp.py
@@ -314,6 +314,21 @@ Subject: This should be accepted.
         items = get_queue_messages('in', expected_count=1)
         self.assertEqual(items[0].msgdata['listid'], 'test.example.com')
 
+    def test_mailing_list_with_utf8_local_part_sender(self):
+        with transaction():
+            create_list('testing@example.com')
+        self.assertEqual(self._mlist.posting_address, 'testing@example.com')
+        self._lmtp.sendmail('raphaël@example.com', ['testing@example.com'], """\
+From: raphaël@example.com
+To: testing@example.com
+Message-ID: <ant>
+Subject: This should be accepted.
+
+""")
+        # The message is in the incoming queue but not the command queue.
+        items = get_queue_messages('in', expected_count=1)
+        self.assertEqual(items[0].msgdata['listid'], 'testing.example.com')
+
 
 class TestBugs(unittest.TestCase):
     """Test some LMTP related bugs."""
@@ -358,7 +373,7 @@ Message-ID: <alpha>
         # Local parts > 64 bytes should be accepted.
         with transaction():
             create_list('longer_than_15_bytes@example.com')
-        recip = 'longer_than_15_bytes-confirm+{}@example.com'.format(40*'x')
+        recip = 'longer_than_15_bytes-confirm+{}@example.com'.format(40 * 'x')
         self._lmtp.sendmail('anne@example.com', [recip], """\
 From: anne@example.com
 To: {}
diff --git a/src/mailman/utilities/mailbox.py b/src/mailman/utilities/mailbox.py
index 59ca06530c..7f23ccbf87 100644
--- a/src/mailman/utilities/mailbox.py
+++ b/src/mailman/utilities/mailbox.py
@@ -24,8 +24,12 @@
 # for us is that it does no 'From' mangling.
 # mangling.
 
-from mailbox import MMDF
+from mailbox import MMDF, _mboxMMDF, _mboxMMDFMessage, linesep
+from mailbox import _mboxMMDF
 from public import public
+import email
+import email.message
+import email.generator
 
 
 @public
@@ -41,3 +45,42 @@ class Mailbox(MMDF):
         self.unlock()
         # Don't suppress the exception.
         return False
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        from_line = self._file.readline().replace(linesep, b'')
+        string = self._file.read(stop - self._file.tell())
+        msg = self._message_factory(string.replace(linesep, b'\n'))
+        msg.set_from(from_line[5:].decode('utf-8'))
+        return msg
+
+    def _install_message(self, message):
+        """Format a message and blindly write to self._file."""
+        from_line = None
+        if isinstance(message, str):
+            message = self._string_to_bytes(message)
+        if isinstance(message, bytes) and message.startswith(b'From '):
+            newline = message.find(b'\n')
+            if newline != -1:
+                from_line = message[:newline]
+                message = message[newline + 1:]
+            else:
+                from_line = message
+                message = b''
+        elif isinstance(message, _mboxMMDFMessage):
+            author = message.get_from().encode('utf-8')
+            from_line = b'From ' + author
+        elif isinstance(message, email.message.Message):
+            from_line = message.get_unixfrom()  # May be None.
+            if from_line is not None:
+               from_line = from_line.encode('utf-8')
+        if from_line is None:
+            from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode()
+        start = self._file.tell()
+        self._file.write(from_line + linesep)
+        self._dump_message(message, self._file, self._mangle_from_)
+        stop = self._file.tell()
+        return (start, stop)
+
diff --git a/tox.ini b/tox.ini
index ab9656dab6..f27e22f519 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py37,py38,py39,py310,py311}-{nocov,cov,diffcov}{,-mysql,-pg},qa
+envlist = {py37,py38,py39,py310}-{nocov,cov,diffcov}{,-mysql,-pg},qa
 #recreate = True
 skip_missing_interpreters = True
 
@@ -16,6 +16,7 @@ commands =
 #sitepackages = True
 usedevelop = True
 deps =
+     distutils
      flufl.testing>=0.8
      nose2
      cov,diffcov: coverage
-- 
GitLab


From 0ca7fee692126b7ed025ff5d85a0b10c240dc12b Mon Sep 17 00:00:00 2001
From: Michel Bernier <mbernier@cofomo.com>
Date: Thu, 18 May 2023 08:38:38 -0400
Subject: [PATCH 3/3] Modify message for rest API.

---
 src/mailman/email/message.py  |  5 +++--
 src/mailman/email/validate.py | 12 ------------
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py
index cbc57f7f9e..9219cea1e1 100644
--- a/src/mailman/email/message.py
+++ b/src/mailman/email/message.py
@@ -35,6 +35,7 @@ from mailman.interfaces.address import IEmailValidator
 from public import public
 from zope.component import getUtility
 
+from mailman.utilities.ua_utils import transform_email_to_utf8
 
 COMMASPACE = ', '
 
@@ -80,7 +81,7 @@ class Message(email.message.EmailMessage):
         for address in self.senders:
             # This could be None or the empty string.
             if address:
-                return address
+                return transform_email_to_utf8(address)
         return ''
 
     @property
@@ -135,7 +136,7 @@ class Message(email.message.EmailMessage):
                 sender = sender.decode('ascii')
             if not validator.is_valid(sender):
                 continue
-            clean_senders.append(sender)
+            clean_senders.append(transform_email_to_utf8(sender))
         return clean_senders
 
 
diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py
index 2ee1aa3333..802431cff8 100644
--- a/src/mailman/email/validate.py
+++ b/src/mailman/email/validate.py
@@ -25,18 +25,6 @@ from mailman.interfaces.address import (
     IEmailValidator,
     InvalidEmailAddressError,
 )
-from mailman.utilities.email import split_email
-from public import public
-from zope.interface import implementer
-
-
-# What other characters should be allowed?
-_valid_local = re.compile("[-0-9a-z!#$%&'*+./=?@_`{}~]", re.IGNORECASE)
-# Strictly speaking, both ^ and | are allowed and others are allowed in quoted
-# local parts, but this can open the door to certain web exploits so we don't
-# allow them.
-_valid_domain = re.compile('[-a-z0-9]', re.IGNORECASE)
-# These are the only characters allowed in domain parts.
 
 
 @public
-- 
GitLab