Commit 5477a201 authored by Mark Sapiro's avatar Mark Sapiro

Merge branch 'pending' into 'master'

Implement an extra large column type for the pendedkeyvalue.value table.

See merge request mailman/mailman!333
parents e7e8c1d1 2087776d
Pipeline #16390336 failed with stage
in 18 minutes and 44 seconds
......@@ -21,7 +21,8 @@ import unittest
from email import message_from_bytes as mfb
from mailman.app.lifecycle import create_list
from mailman.chains.hold import autorespond_to_sender
from mailman.chains.builtin import BuiltInChain
from mailman.chains.hold import HoldChain, autorespond_to_sender
from mailman.core.chains import process as process_chain
from mailman.interfaces.autorespond import IAutoResponseSet, Response
from mailman.interfaces.member import MemberRole
......@@ -36,6 +37,9 @@ from pkg_resources import resource_filename
from zope.component import getUtility
SEMISPACE = '; '
class TestAutorespond(unittest.TestCase):
"""Test autorespond_to_sender()"""
......@@ -224,3 +228,33 @@ A message body.
messages = list(getUtility(IMessageStore).messages)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['message-id'], '<ant>')
def test_hold_with_long_rule_misses(self):
msg = mfs("""\
From: anne@example.com
To: test@example.com
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
rule_misses = [x[0] for x in BuiltInChain._link_descriptions
if x[0] not in ('truth', 'any')]
for i in range(20):
rule_misses.append('header-match-test.example.com-{}'.format(i))
msgdata = dict(rule_misses=rule_misses)
msgdata['rule_hits'] = []
msgdata['moderation_reasons'] = ['something']
# We can't use process_chain because it clears rule hits and misses.
HoldChain()._process(self._mlist, msg, msgdata)
messages = get_queue_messages('virgin', expected_count=2)
for item in messages:
if item.msg['to'] == 'test-owner@example.com':
held_message = item.msg.get_payload(1).get_payload(0)
elif item.msg['To'] == 'anne@example.com':
pass
else:
self.fail('Unexpected message: %s' % item.msg)
self.assertEqual(held_message['x-mailman-rule-misses'],
SEMISPACE.join(rule_misses))
"""further_increase_pendedkeyvalue_value_size
Revision ID: 3f31035ed0d7
Revises: 4bd95c99b2e7
Create Date: 2017-10-18 17:42:35.550686
"""
from alembic import op
from mailman.database.types import SAUnicodeLarge, SAUnicodeXL
# revision identifiers, used by Alembic.
revision = '3f31035ed0d7'
down_revision = '4bd95c99b2e7'
def upgrade():
# pendedkeyvalue table values can be much larger than SAUnicodeLarge
with op.batch_alter_table('pendedkeyvalue') as batch_op:
# Drop the existing index on the table.
batch_op.drop_index(op.f('ix_pendedkeyvalue_value'))
# Alter the column type and then create a new index with
# mysql_length set to a fixed length value.
batch_op.alter_column('value', type_=SAUnicodeXL)
batch_op.create_index(op.f('ix_pendedkeyvalue_value'),
columns=['value'], mysql_length=100)
def downgrade():
with op.batch_alter_table('pendedkeyvalue') as batch_op:
batch_op.alter_column('value', type_=SAUnicodeLarge)
# Drop the existing index because it has a fixed length value and then
# re-create without the length constraint.
batch_op.drop_index(op.f('ix_pendedkeyvalue_value'))
batch_op.create_index(op.f('ix_pendedkeyvalue_value'),
columns=['value'])
......@@ -124,3 +124,29 @@ def compile_sa_unicode_large(element, compiler, **kw):
@compiles(SAUnicodeLarge)
def default_sa_unicode_large(element, compiler, **kw):
return compiler.visit_unicode(element, **kw)
@public
class SAUnicodeXL(TypeDecorator):
"""Similar to SAUnicode type, but compiles to VARCHAR(20000).
This allows very long values up to 20,000 bytes for cases where
SAUnicodeLarge is not enough.
Because of MySQL issues columns of this type should either not be indexed
or should be indexed with an Index() function specifying mysql_length=N
where N is (probably much) less than 767.
See http://docs.sqlalchemy.org/en/latest/dialects/mysql.html#index-length
"""
impl = Unicode
@compiles(SAUnicodeXL, 'mysql')
def compile_sa_unicode_xl(element, compiler, **kw):
# We hardcode the collate here to make string comparison case sensitive.
return 'VARCHAR(20000) COLLATE utf8_bin' # pragma: nocover
@compiles(SAUnicodeXL)
def default_sa_unicode_xl(element, compiler, **kw):
return compiler.visit_unicode(element, **kw)
......@@ -49,6 +49,9 @@ Bugs
* The email.message.UserNotification() constructor will now not fail if given
a ``text`` argument containing a character not in the charset of the
``lang`` argument. (Closes #437)
* A new SQLAlchemy column type ``SAUnicodeXL`` has been implemented to support
large columns in MySQL and is used for the ``value`` column of the
``pendedkeyvalue`` table. (Closes #385)
Command line
------------
......
......@@ -23,13 +23,13 @@ from lazr.config import as_timedelta
from mailman.config import config
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
from mailman.database.types import SAUnicode, SAUnicodeLarge
from mailman.database.types import SAUnicode, SAUnicodeXL
from mailman.interfaces.pending import (
IPendable, IPended, IPendedKeyValue, IPendings)
from mailman.utilities.datetime import now
from mailman.utilities.uid import TokenFactory
from public import public
from sqlalchemy import Column, DateTime, ForeignKey, Integer, and_
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, and_
from sqlalchemy.orm import aliased, relationship
from zope.interface import implementer
from zope.interface.verify import verifyObject
......@@ -44,10 +44,12 @@ class PendedKeyValue(Model):
"""A pended key/value pair, tied to a token."""
__tablename__ = 'pendedkeyvalue'
__table_args__ = (Index('ix_pendedkeyvalue_value', 'value',
mysql_length=100), )
id = Column(Integer, primary_key=True)
key = Column(SAUnicode, index=True)
value = Column(SAUnicodeLarge, index=True)
value = Column(SAUnicodeXL)
pended_id = Column(Integer, ForeignKey('pended.id'), index=True)
def __init__(self, key, value):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment