Commit b4d3a036 authored by Barry Warsaw's avatar Barry Warsaw

* The `bounceevent` table now uses list-ids to cross-reference the mailing

   list, to match other tables.  Similarly for the `IBounceEvent` interface.

Also:

- Move the acquisition of the database lock during creation to the
  IDatabaseFactory.create() method instead of the individual database
  initialize() methods.

- In the migration.rst doctest, don't delete teh version records when using
  SQLite, since that breaks tests.

- Implement a few nice helpers for database migrations, including
  make_listid() for turning a list name into a list id, and pivot() which
  simplifies moving the backup table to the final table name.
parents d370397e b36b316a
......@@ -2,7 +2,7 @@
Bounces
=======
An important feature of Mailman is automatic bounce process.
An important feature of Mailman is automatic bounce processing.
Bounces, or message rejection
......
......@@ -27,7 +27,6 @@ import os
import sys
import logging
from flufl.lock import Lock
from lazr.config import as_boolean
from pkg_resources import resource_listdir, resource_string
from storm.cache import GenerationalCache
......@@ -60,13 +59,6 @@ class StormBaseDatabase:
self.url = None
self.store = None
def initialize(self, debug=None):
"""See `IDatabase`."""
# Serialize this so we don't get multiple processes trying to create
# the database at the same time.
with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
self._create(debug)
def begin(self):
"""See `IDatabase`."""
# Storm takes care of this for us.
......@@ -117,7 +109,8 @@ class StormBaseDatabase:
"""
pass
def _create(self, debug):
def initialize(self, debug=None):
"""See `IDatabase`."""
# Calculate the engine url.
url = expand(config.database.url, config.paths)
log.debug('Database url: %s', url)
......
......@@ -30,13 +30,14 @@ already applied.
>>> from mailman.model.version import Version
>>> results = config.db.store.find(Version, component='schema')
>>> results.count()
3
4
>>> versions = sorted(result.version for result in results)
>>> for version in versions:
... print version
00000000000000
20120407000000
20121015000000
20130406000000
Migrations
......@@ -79,7 +80,7 @@ This migration module just adds a marker to the `version` table.
>>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
... pass
>>> with open(os.path.join(path, 'mm_20129999000000.py'), 'w') as fp:
>>> with open(os.path.join(path, 'mm_20159999000000.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
......@@ -98,14 +99,15 @@ This will load the new migration, since it hasn't been loaded before.
00000000000000
20120407000000
20121015000000
20129999000000
20130406000000
20159999000000
>>> test = config.db.store.find(Version, component='test').one()
>>> print test.version
20129999000000
20159999000000
Migrations will only be loaded once.
>>> with open(os.path.join(path, 'mm_20129999000001.py'), 'w') as fp:
>>> with open(os.path.join(path, 'mm_20159999000001.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
......@@ -129,13 +131,14 @@ The first time we load this new migration, we'll get the 801 marker.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20130406000000
20159999000000
20159999000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20129999000000
20159999000000
We do not get an 802 marker because the migration has already been loaded.
......@@ -146,13 +149,14 @@ We do not get an 802 marker because the migration has already been loaded.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20130406000000
20159999000000
20159999000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20129999000000
20159999000000
Partial upgrades
......@@ -165,13 +169,13 @@ additional migrations, intended to be applied in sequential order.
>>> from shutil import copyfile
>>> from mailman.testing.helpers import chdir
>>> with chdir(path):
... copyfile('mm_20129999000000.py', 'mm_20129999000002.py')
... copyfile('mm_20129999000000.py', 'mm_20129999000003.py')
... copyfile('mm_20129999000000.py', 'mm_20129999000004.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000002.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000003.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000004.py')
Now, only migrate to the ...03 timestamp.
>>> config.db.load_migrations('20129999000003')
>>> config.db.load_migrations('20159999000003')
You'll notice that the ...04 version is not present.
......@@ -181,10 +185,11 @@ You'll notice that the ...04 version is not present.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20129999000002
20129999000003
20130406000000
20159999000000
20159999000001
20159999000002
20159999000003
.. cleanup:
......@@ -194,8 +199,9 @@ You'll notice that the ...04 version is not present.
this will cause migration.rst to fail on subsequent runs. So let's just
clean up the database explicitly.
>>> results = config.db.store.execute("""
... DELETE FROM version WHERE version.version >= '201299990000'
... OR version.component = 'test';
... """)
>>> config.db.commit()
>>> if config.db.TAG != 'sqlite':
... results = config.db.store.execute("""
... DELETE FROM version WHERE version.version >= '201299990000'
... OR version.component = 'test';
... """)
... config.db.commit()
......@@ -27,8 +27,10 @@ __all__ = [
]
import os
import types
from flufl.lock import Lock
from zope.component import getAdapter
from zope.interface import implementer
from zope.interface.verify import verifyObject
......@@ -47,13 +49,14 @@ class DatabaseFactory:
@staticmethod
def create():
"""See `IDatabaseFactory`."""
database_class = config.database['class']
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
database.load_migrations()
database.commit()
return database
with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
database_class = config.database['class']
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
database.load_migrations()
database.commit()
return database
......
# Copyright (C) 2013 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""Schema migration helpers."""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'make_listid',
]
def make_listid(fqdn_listname):
"""Turn a FQDN list name into a List-ID."""
list_name, at, mail_host = fqdn_listname.partition('@')
if at == '':
# If there is no @ sign in the value, assume it already contains the
# list-id.
return fqdn_listname
return '{0}.{1}'.format(list_name, mail_host)
def pivot(store, table_name):
"""Pivot a backup table into the real table name."""
store.execute('DROP TABLE {}'.format(table_name))
store.execute('ALTER TABLE {0}_backup RENAME TO {0}'.format(table_name))
......@@ -48,11 +48,11 @@ __all__ = [
]
from mailman.database.schema.helpers import pivot
from mailman.interfaces.archiver import ArchivePolicy
VERSION = '20120407000000'
_helper = None
......@@ -98,7 +98,7 @@ def upgrade_sqlite(database, store, version, module_path):
list_id = '{0}.{1}'.format(list_name, mail_host)
fqdn_listname = '{0}@{1}'.format(list_name, mail_host)
store.execute("""
UPDATE ml_backup SET
UPDATE mailinglist_backup SET
allow_list_posts = {0},
newsgroup_moderation = {1},
nntp_prefix_subject_too = {2},
......@@ -120,8 +120,7 @@ def upgrade_sqlite(database, store, version, module_path):
WHERE mailing_list = '{1}';
""".format(list_id, fqdn_listname))
# Pivot the backup table to the real thing.
store.execute('DROP TABLE mailinglist;')
store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
pivot(store, 'mailinglist')
# Now add some indexes that were previously missing.
store.execute(
'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
......@@ -137,12 +136,11 @@ def upgrade_sqlite(database, store, version, module_path):
else:
list_id = '{0}.{1}'.format(list_name, mail_host)
store.execute("""
UPDATE mem_backup SET list_id = '{0}'
UPDATE member_backup SET list_id = '{0}'
WHERE id = {1};
""".format(list_id, id))
# Pivot the backup table to the real thing.
store.execute('DROP TABLE member;')
store.execute('ALTER TABLE mem_backup RENAME TO member;')
pivot(store, 'member')
......
......@@ -33,6 +33,9 @@ __all__ = [
]
from mailman.database.schema.helpers import make_listid, pivot
VERSION = '20121015000000'
......@@ -44,20 +47,10 @@ def upgrade(database, store, version, module_path):
upgrade_postgres(database, store, version, module_path)
def _make_listid(fqdn_listname):
list_name, at, mail_host = fqdn_listname.partition('@')
if at == '':
# If there is no @ sign in the value, assume it already contains the
# list-id.
return fqdn_listname
return '{0}.{1}'.format(list_name, mail_host)
def upgrade_sqlite(database, store, version, module_path):
database.load_schema(
store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
store, version, 'sqlite_{}_01.sql'.format(version), module_path)
results = store.execute("""
SELECT id, mailing_list
FROM ban;
......@@ -67,15 +60,12 @@ def upgrade_sqlite(database, store, version, module_path):
if mailing_list is None:
continue
store.execute("""
UPDATE ban_backup SET list_id = '{0}'
WHERE id = {1};
""".format(_make_listid(mailing_list), id))
UPDATE ban_backup SET list_id = '{}'
WHERE id = {};
""".format(make_listid(mailing_list), id))
# Pivot the bans backup table to the real thing.
store.execute('DROP TABLE ban;')
store.execute('ALTER TABLE ban_backup RENAME TO ban;')
# Pivot the mailinglist backup table to the real thing.
store.execute('DROP TABLE mailinglist;')
store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
pivot(store, 'ban')
pivot(store, 'mailinglist')
......@@ -90,7 +80,7 @@ def upgrade_postgres(database, store, version, module_path):
store.execute("""
UPDATE ban SET list_id = '{0}'
WHERE id = {1};
""".format(_make_listid(mailing_list), id))
""".format(make_listid(mailing_list), id))
store.execute('ALTER TABLE ban DROP COLUMN mailing_list;')
store.execute('ALTER TABLE mailinglist DROP COLUMN new_member_options;')
store.execute('ALTER TABLE mailinglist DROP COLUMN send_reminders;')
......
# Copyright (C) 2013 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""3.0b3 -> 3.0b4 schema migrations.
Renamed:
* bounceevent.list_name -> bounceevent.list_id
"""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'upgrade'
]
from mailman.database.schema.helpers import make_listid, pivot
VERSION = '20130406000000'
def upgrade(database, store, version, module_path):
if database.TAG == 'sqlite':
upgrade_sqlite(database, store, version, module_path)
else:
upgrade_postgres(database, store, version, module_path)
def upgrade_sqlite(database, store, version, module_path):
database.load_schema(
store, version, 'sqlite_{}_01.sql'.format(version), module_path)
results = store.execute("""
SELECT id, list_name
FROM bounceevent;
""")
for id, list_name in results:
store.execute("""
UPDATE bounceevent_backup SET list_id = '{}'
WHERE id = {};
""".format(make_listid(list_name), id))
pivot(store, 'bounceevent')
def upgrade_postgres(database, store, version, module_path):
pass
-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
-- This file contains the sqlite3 schema migration from
-- 3.0b1 TO 3.0b2
--
-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE.
-- 3.0b2 has been released thus you MAY NOT edit this file.
-- For SQLite3 migration strategy, see
-- http://sqlite.org/faq.html#q11
-- REMOVALS from the mailinglist table.
-- REMOVALS from the mailinglist table:
-- REM archive
-- REM archive_private
-- REM archive_volume_frequency
......@@ -15,7 +15,7 @@
-- REM news_prefix_subject_too
-- REM nntp_host
--
-- ADDS to the mailing list table.
-- ADDS to the mailing list table:
-- ADD allow_list_posts
-- ADD archive_policy
-- ADD list_id
......@@ -25,16 +25,16 @@
-- LP: #971013
-- LP: #967238
-- REMOVALS from the member table.
-- REMOVALS from the member table:
-- REM mailing_list
-- ADDS to the member table.
-- ADDS to the member table:
-- ADD list_id
-- LP: #1024509
CREATE TABLE ml_backup (
CREATE TABLE mailinglist_backup (
id INTEGER NOT NULL,
-- List identity
list_name TEXT,
......@@ -142,7 +142,7 @@ CREATE TABLE ml_backup (
PRIMARY KEY (id)
);
INSERT INTO ml_backup SELECT
INSERT INTO mailinglist_backup SELECT
id,
-- List identity
list_name,
......@@ -249,7 +249,7 @@ INSERT INTO ml_backup SELECT
welcome_message_uri
FROM mailinglist;
CREATE TABLE mem_backup(
CREATE TABLE member_backup(
id INTEGER NOT NULL,
_member_id TEXT,
role INTEGER,
......@@ -260,7 +260,7 @@ CREATE TABLE mem_backup(
PRIMARY KEY (id)
);
INSERT INTO mem_backup SELECT
INSERT INTO member_backup SELECT
id,
_member_id,
role,
......@@ -272,9 +272,9 @@ INSERT INTO mem_backup SELECT
-- Add the new columns. They'll get inserted at the Python layer.
ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER;
ALTER TABLE ml_backup ADD COLUMN list_id TEXT;
ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
ALTER TABLE ml_backup ADD COLUMN newsgroup_moderation INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN archive_policy INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN list_id TEXT;
ALTER TABLE mailinglist_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN newsgroup_moderation INTEGER;
ALTER TABLE mem_backup ADD COLUMN list_id TEXT;
ALTER TABLE member_backup ADD COLUMN list_id TEXT;
-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
-- This file contains the sqlite3 schema migration from
-- 3.0b2 TO 3.0b3
--
-- AFTER 3.0b3 IS RELEASED YOU MAY NOT EDIT THIS FILE.
-- 3.0b3 has been released thus you MAY NOT edit this file.
-- REMOVALS from the ban table.
-- REMOVALS from the ban table:
-- REM mailing_list
-- ADDS to the ban table.
-- ADDS to the ban table:
-- ADD list_id
CREATE TABLE ban_backup (
......@@ -30,7 +30,7 @@ ALTER TABLE ban_backup ADD COLUMN list_id TEXT;
-- REM private_roster
-- REM admin_member_chunksize
CREATE TABLE ml_backup (
CREATE TABLE mailinglist_backup (
id INTEGER NOT NULL,
list_name TEXT,
mail_host TEXT,
......@@ -130,7 +130,7 @@ CREATE TABLE ml_backup (
PRIMARY KEY (id)
);
INSERT INTO ml_backup SELECT
INSERT INTO mailinglist_backup SELECT
id,
list_name,
mail_host,
......
-- This file contains the SQLite schema migration from
-- 3.0b3 to 3.0b4
--
-- After 3.0b4 is released you may not edit this file.
-- For SQLite3 migration strategy, see
-- http://sqlite.org/faq.html#q11
-- REMOVALS from the bounceevent table:
-- REM list_name
-- ADDS to the ban bounceevent table:
-- ADD list_id
CREATE TABLE bounceevent_backup (
id INTEGER NOT NULL,
email TEXT,
'timestamp' TIMESTAMP,
message_id TEXT,
context INTEGER,
processed BOOLEAN,
PRIMARY KEY (id)
);
INSERT INTO bounceevent_backup SELECT
id, email, "timestamp", message_id,
context, processed
FROM bounceevent;
ALTER TABLE bounceevent_backup ADD COLUMN list_id TEXT;
......@@ -26,11 +26,14 @@ __all__ = [
'TestMigration20120407UnchangedData',
'TestMigration20121015MigratedData',
'TestMigration20121015Schema',
'TestMigration20130406MigratedData',
'TestMigration20130406Schema',
]
import unittest
from datetime import datetime
from operator import attrgetter
from pkg_resources import resource_string
from storm.exceptions import DatabaseError
......@@ -39,11 +42,13 @@ from zope.component import getUtility
from mailman.interfaces.database import IDatabaseFactory
from mailman.interfaces.domain import IDomainManager
from mailman.interfaces.archiver import ArchivePolicy
from mailman.interfaces.bounce import BounceContext
from mailman.interfaces.listmanager import IListManager
from mailman.interfaces.mailinglist import IAcceptableAliasSet
from mailman.interfaces.nntp import NewsgroupModeration
from mailman.interfaces.subscriptions import ISubscriptionService
from mailman.model.bans import Ban
from mailman.model.bounce import BounceEvent
from mailman.testing.helpers import temporary_db
from mailman.testing.layers import ConfigLayer
......@@ -426,3 +431,49 @@ class TestMigration20121015MigratedData(MigrationTestBase):
self.assertEqual(bans[0].list_id, 'test.example.com')
self.assertEqual(bans[1].email, '[email protected]')
self.assertEqual(bans[1].list_id, None)
class TestMigration20130406Schema(MigrationTestBase):
"""Test column migrations."""
def test_pre_upgrade_column_migrations(self):
self._missing_present('bounceevent',
['20130405999999'],
('list_id',),
('list_name',))
def test_post_upgrade_column_migrations(self):
self._missing_present('bounceevent',
['20130405999999',
'20130406000000'],
('list_name',),
('list_id',))
class TestMigration20130406MigratedData(MigrationTestBase):
"""Test migrated data."""
def test_migration_bounceevent(self):
# Load all migrations to just before the one we're testing.
self._database.load_migrations('20130405999999')
# Insert a bounce event.
self._database.store.execute("""
INSERT INTO bounceevent VALUES (
1, '[email protected]', '[email protected]',
'2013-04-06 21:12:00', '<[email protected]>',
1, 0);
""")
# Update to the current migration we're testing
self._database.load_migrations('20130406000000')
# The bounce event should exist, but with a list-id instead of a fqdn
# list name.
events = list(self._database.store.find(BounceEvent))
self.assertEqual(len(events), 1)
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].email, '[email protected]')
self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
self.assertEqual(events[0].message_id, '<[email protected]>')
self.assertEqual(events[0].context, BounceContext.normal)
self.assertFalse(events[0].processed)
......@@ -43,6 +43,11 @@ Configuration
the configuration file, just after ``./mailman.cfg`` and before
``~/.mailman.cfg``. (LP: #1157861)
Database
--------
* The `bounceevent` table now uses list-ids to cross-reference the mailing
list, to match other tables. Similarly for the `IBounceEvent` interface.
Bugs
----
* Non-queue runners should not create ``var/queue`` subdirectories. Fixed by
......
......@@ -60,8 +60,8 @@ class UnrecognizedBounceDisposition(Enum):
class IBounceEvent(Interface):
"""Registration record for a single bounce event."""
list_name = Attribute(
"""The name of the mailing list that received this bounce.""")
list_id = Attribute(
"""The List-ID of the mailing list that received this bounce.""")
email = Attribute(
"""The email address that bounced.""")
......
......@@ -43,15 +43,15 @@ class BounceEvent(Model):
"""See `IBounceEvent`."""
id = Int(primary=True)
list_name = Unicode()
list_id = Unicode()
email = Unicode()
timestamp = DateTime()
message_id = Unicode()
context = Enum(BounceContext)
processed = Bool()
def __init__(self, list_name, email, msg, context=None):
self.list_name = list_name
def __init__(self, list_id, email, msg, context=None):
self.list_id = list_id
self.email = email
self.timestamp = now()
self.message_id = msg['message-id']
......@@ -67,7 +67,7 @@ class BounceProcessor:
@dbconnection
def register(self, store, mlist, email, msg, where=None):
"""See `IBounceProcessor`."""
event = BounceEvent(mlist.fqdn_listname, email, msg, where)
event = BounceEvent(mlist.list_id, email, msg, where)
store.add(event)
return event
......
......@@ -39,8 +39,8 @@ of bouncing email addresses. These are passed one-by-one to the registration
interface.
>>> event = processor.register(mlist, '[email protected]', msg)
>>> print event.list_name
test@example.com
>>> print event.list_id
test.example.com
>>> print event.email
[email protected]
>>> print event.message_id
......
......@@ -58,7 +58,7 @@ Message-Id: <first>
events = list(self._processor.events)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event.list_name, '[email protected]example.com')
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '[email protected]')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
......@@ -68,7 +68,7 @@ Message-Id: <first>
unprocessed = list(self._processor.unprocessed)
self.assertEqual(len(unprocessed), 1)
event = unprocessed[0]
self.assertEqual(event.list_name, '[email protected]example.com')
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '[email protected]')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
......