Commit d146f14b authored by Barry Warsaw's avatar Barry Warsaw

Migrate bounceevent.list_name -> bounceevent.list_id

* Rename StormBaseDatabase._create() -> .initialize()
* Refactor database initialization.
* make_listid() helper.
* Add a pivot() helper for schema migrations.
parent 41059ed2
......@@ -27,7 +27,6 @@ import os
import sys
import logging
from flufl.lock import Lock
from lazr.config import as_boolean
from pkg_resources import resource_listdir, resource_string
from storm.cache import GenerationalCache
......@@ -60,13 +59,6 @@ class StormBaseDatabase:
self.url = None
self.store = None
def initialize(self, debug=None):
"""See `IDatabase`."""
# Serialize this so we don't get multiple processes trying to create
# the database at the same time.
with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
self._create(debug)
def begin(self):
"""See `IDatabase`."""
# Storm takes care of this for us.
......@@ -117,7 +109,8 @@ class StormBaseDatabase:
"""
pass
def _create(self, debug):
def initialize(self, debug=None):
"""See `IDatabase`."""
# Calculate the engine url.
url = expand(config.database.url, config.paths)
log.debug('Database url: %s', url)
......
......@@ -30,13 +30,14 @@ already applied.
>>> from mailman.model.version import Version
>>> results = config.db.store.find(Version, component='schema')
>>> results.count()
3
4
>>> versions = sorted(result.version for result in results)
>>> for version in versions:
... print version
00000000000000
20120407000000
20121015000000
20130406000000
Migrations
......@@ -79,7 +80,7 @@ This migration module just adds a marker to the `version` table.
>>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
... pass
>>> with open(os.path.join(path, 'mm_20129999000000.py'), 'w') as fp:
>>> with open(os.path.join(path, 'mm_20159999000000.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
......@@ -98,14 +99,15 @@ This will load the new migration, since it hasn't been loaded before.
00000000000000
20120407000000
20121015000000
20129999000000
20130406000000
20159999000000
>>> test = config.db.store.find(Version, component='test').one()
>>> print test.version
20129999000000
20159999000000
Migrations will only be loaded once.
>>> with open(os.path.join(path, 'mm_20129999000001.py'), 'w') as fp:
>>> with open(os.path.join(path, 'mm_20159999000001.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
......@@ -129,13 +131,14 @@ The first time we load this new migration, we'll get the 801 marker.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20130406000000
20159999000000
20159999000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20129999000000
20159999000000
We do not get an 802 marker because the migration has already been loaded.
......@@ -146,13 +149,14 @@ We do not get an 802 marker because the migration has already been loaded.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20130406000000
20159999000000
20159999000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20129999000000
20159999000000
Partial upgrades
......@@ -165,13 +169,13 @@ additional migrations, intended to be applied in sequential order.
>>> from shutil import copyfile
>>> from mailman.testing.helpers import chdir
>>> with chdir(path):
... copyfile('mm_20129999000000.py', 'mm_20129999000002.py')
... copyfile('mm_20129999000000.py', 'mm_20129999000003.py')
... copyfile('mm_20129999000000.py', 'mm_20129999000004.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000002.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000003.py')
... copyfile('mm_20159999000000.py', 'mm_20159999000004.py')
Now, only migrate to the ...03 timestamp.
>>> config.db.load_migrations('20129999000003')
>>> config.db.load_migrations('20159999000003')
You'll notice that the ...04 version is not present.
......@@ -181,10 +185,11 @@ You'll notice that the ...04 version is not present.
00000000000000
20120407000000
20121015000000
20129999000000
20129999000001
20129999000002
20129999000003
20130406000000
20159999000000
20159999000001
20159999000002
20159999000003
.. cleanup:
......
......@@ -27,8 +27,10 @@ __all__ = [
]
import os
import types
from flufl.lock import Lock
from zope.component import getAdapter
from zope.interface import implementer
from zope.interface.verify import verifyObject
......@@ -47,13 +49,15 @@ class DatabaseFactory:
@staticmethod
def create():
"""See `IDatabaseFactory`."""
database_class = config.database['class']
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
database.load_migrations()
database.commit()
return database
with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
database_class = config.database['class']
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
database.load_migrations()
database.commit()
import sys; print('db -> done', os.getpid(), file=sys.stderr)
return database
......
# Copyright (C) 2013 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""Schema migration helpers."""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'make_listid',
]
def make_listid(fqdn_listname):
"""Turn a FQDN list name into a List-ID."""
list_name, at, mail_host = fqdn_listname.partition('@')
if at == '':
# If there is no @ sign in the value, assume it already contains the
# list-id.
return fqdn_listname
return '{0}.{1}'.format(list_name, mail_host)
def pivot(store, table_name):
"""Pivot a backup table into the real table name."""
store.execute('DROP TABLE {}'.format(table_name))
store.execute('ALTER TABLE {0}_backup RENAME TO {0}'.format(table_name))
......@@ -48,11 +48,11 @@ __all__ = [
]
from mailman.database.schema.helpers import pivot
from mailman.interfaces.archiver import ArchivePolicy
VERSION = '20120407000000'
_helper = None
......@@ -98,7 +98,7 @@ def upgrade_sqlite(database, store, version, module_path):
list_id = '{0}.{1}'.format(list_name, mail_host)
fqdn_listname = '{0}@{1}'.format(list_name, mail_host)
store.execute("""
UPDATE ml_backup SET
UPDATE mailinglist_backup SET
allow_list_posts = {0},
newsgroup_moderation = {1},
nntp_prefix_subject_too = {2},
......@@ -120,8 +120,7 @@ def upgrade_sqlite(database, store, version, module_path):
WHERE mailing_list = '{1}';
""".format(list_id, fqdn_listname))
# Pivot the backup table to the real thing.
store.execute('DROP TABLE mailinglist;')
store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
pivot(store, 'mailinglist')
# Now add some indexes that were previously missing.
store.execute(
'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
......@@ -137,12 +136,11 @@ def upgrade_sqlite(database, store, version, module_path):
else:
list_id = '{0}.{1}'.format(list_name, mail_host)
store.execute("""
UPDATE mem_backup SET list_id = '{0}'
UPDATE member_backup SET list_id = '{0}'
WHERE id = {1};
""".format(list_id, id))
# Pivot the backup table to the real thing.
store.execute('DROP TABLE member;')
store.execute('ALTER TABLE mem_backup RENAME TO member;')
pivot(store, 'member')
......
......@@ -33,6 +33,9 @@ __all__ = [
]
from mailman.database.schema.helpers import make_listid, pivot
VERSION = '20121015000000'
......@@ -44,20 +47,10 @@ def upgrade(database, store, version, module_path):
upgrade_postgres(database, store, version, module_path)
def _make_listid(fqdn_listname):
list_name, at, mail_host = fqdn_listname.partition('@')
if at == '':
# If there is no @ sign in the value, assume it already contains the
# list-id.
return fqdn_listname
return '{0}.{1}'.format(list_name, mail_host)
def upgrade_sqlite(database, store, version, module_path):
database.load_schema(
store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
store, version, 'sqlite_{}_01.sql'.format(version), module_path)
results = store.execute("""
SELECT id, mailing_list
FROM ban;
......@@ -67,15 +60,12 @@ def upgrade_sqlite(database, store, version, module_path):
if mailing_list is None:
continue
store.execute("""
UPDATE ban_backup SET list_id = '{0}'
WHERE id = {1};
""".format(_make_listid(mailing_list), id))
UPDATE ban_backup SET list_id = '{}'
WHERE id = {};
""".format(make_listid(mailing_list), id))
# Pivot the bans backup table to the real thing.
store.execute('DROP TABLE ban;')
store.execute('ALTER TABLE ban_backup RENAME TO ban;')
# Pivot the mailinglist backup table to the real thing.
store.execute('DROP TABLE mailinglist;')
store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
pivot(store, 'ban')
pivot(store, 'mailinglist')
......@@ -90,7 +80,7 @@ def upgrade_postgres(database, store, version, module_path):
store.execute("""
UPDATE ban SET list_id = '{0}'
WHERE id = {1};
""".format(_make_listid(mailing_list), id))
""".format(make_listid(mailing_list), id))
store.execute('ALTER TABLE ban DROP COLUMN mailing_list;')
store.execute('ALTER TABLE mailinglist DROP COLUMN new_member_options;')
store.execute('ALTER TABLE mailinglist DROP COLUMN send_reminders;')
......
# Copyright (C) 2013 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""3.0b3 -> 3.0b4 schema migrations.
Renamed:
* bounceevent.list_name -> bounceevent.list_id
"""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'upgrade'
]
from mailman.database.schema.helpers import make_listid, pivot
VERSION = '20130406000000'
def upgrade(database, store, version, module_path):
if database.TAG == 'sqlite':
upgrade_sqlite(database, store, version, module_path)
else:
upgrade_postgres(database, store, version, module_path)
def upgrade_sqlite(database, store, version, module_path):
database.load_schema(
store, version, 'sqlite_{}_01.sql'.format(version), module_path)
results = store.execute("""
SELECT id, list_name
FROM bounceevent;
""")
for id, list_name in results:
store.execute("""
UPDATE bounceevent_backup SET list_id = '{}'
WHERE id = {};
""".format(make_listid(list_name), id))
pivot(store, 'bounceevent')
def upgrade_postgres(database, store, version, module_path):
pass
-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
-- This file contains the sqlite3 schema migration from
-- 3.0b1 TO 3.0b2
--
-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE.
-- 3.0b2 has been released thus you MAY NOT edit this file.
-- For SQLite3 migration strategy, see
-- http://sqlite.org/faq.html#q11
-- REMOVALS from the mailinglist table.
-- REMOVALS from the mailinglist table:
-- REM archive
-- REM archive_private
-- REM archive_volume_frequency
......@@ -15,7 +15,7 @@
-- REM news_prefix_subject_too
-- REM nntp_host
--
-- ADDS to the mailing list table.
-- ADDS to the mailing list table:
-- ADD allow_list_posts
-- ADD archive_policy
-- ADD list_id
......@@ -25,16 +25,16 @@
-- LP: #971013
-- LP: #967238
-- REMOVALS from the member table.
-- REMOVALS from the member table:
-- REM mailing_list
-- ADDS to the member table.
-- ADDS to the member table:
-- ADD list_id
-- LP: #1024509
CREATE TABLE ml_backup (
CREATE TABLE mailinglist_backup (
id INTEGER NOT NULL,
-- List identity
list_name TEXT,
......@@ -142,7 +142,7 @@ CREATE TABLE ml_backup (
PRIMARY KEY (id)
);
INSERT INTO ml_backup SELECT
INSERT INTO mailinglist_backup SELECT
id,
-- List identity
list_name,
......@@ -249,7 +249,7 @@ INSERT INTO ml_backup SELECT
welcome_message_uri
FROM mailinglist;
CREATE TABLE mem_backup(
CREATE TABLE member_backup(
id INTEGER NOT NULL,
_member_id TEXT,
role INTEGER,
......@@ -260,7 +260,7 @@ CREATE TABLE mem_backup(
PRIMARY KEY (id)
);
INSERT INTO mem_backup SELECT
INSERT INTO member_backup SELECT
id,
_member_id,
role,
......@@ -272,9 +272,9 @@ INSERT INTO mem_backup SELECT
-- Add the new columns. They'll get inserted at the Python layer.
ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER;
ALTER TABLE ml_backup ADD COLUMN list_id TEXT;
ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
ALTER TABLE ml_backup ADD COLUMN newsgroup_moderation INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN archive_policy INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN list_id TEXT;
ALTER TABLE mailinglist_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
ALTER TABLE mailinglist_backup ADD COLUMN newsgroup_moderation INTEGER;
ALTER TABLE mem_backup ADD COLUMN list_id TEXT;
ALTER TABLE member_backup ADD COLUMN list_id TEXT;
-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
-- This file contains the sqlite3 schema migration from
-- 3.0b2 TO 3.0b3
--
-- AFTER 3.0b3 IS RELEASED YOU MAY NOT EDIT THIS FILE.
-- 3.0b3 has been released thus you MAY NOT edit this file.
-- REMOVALS from the ban table.
-- REMOVALS from the ban table:
-- REM mailing_list
-- ADDS to the ban table.
-- ADDS to the ban table:
-- ADD list_id
CREATE TABLE ban_backup (
......@@ -30,7 +30,7 @@ ALTER TABLE ban_backup ADD COLUMN list_id TEXT;
-- REM private_roster
-- REM admin_member_chunksize
CREATE TABLE ml_backup (
CREATE TABLE mailinglist_backup (
id INTEGER NOT NULL,
list_name TEXT,
mail_host TEXT,
......@@ -130,7 +130,7 @@ CREATE TABLE ml_backup (
PRIMARY KEY (id)
);
INSERT INTO ml_backup SELECT
INSERT INTO mailinglist_backup SELECT
id,
list_name,
mail_host,
......
-- This file contains the SQLite schema migration from
-- 3.0b3 to 3.0b4
--
-- After 3.0b4 is released you may not edit this file.
-- For SQLite3 migration strategy, see
-- http://sqlite.org/faq.html#q11
-- REMOVALS from the bounceevent table:
-- REM list_name
-- ADDS to the ban bounceevent table:
-- ADD list_id
CREATE TABLE bounceevent_backup (
id INTEGER NOT NULL,
email TEXT,
'timestamp' TIMESTAMP,
message_id TEXT,
context INTEGER,
processed BOOLEAN,
PRIMARY KEY (id)
);
INSERT INTO bounceevent_backup SELECT
id, email, "timestamp", message_id,
context, processed
FROM bounceevent;
ALTER TABLE bounceevent_backup ADD COLUMN list_id TEXT;
......@@ -26,11 +26,14 @@ __all__ = [
'TestMigration20120407UnchangedData',
'TestMigration20121015MigratedData',
'TestMigration20121015Schema',
'TestMigration20130406MigratedData',
'TestMigration20130406Schema',
]
import unittest
from datetime import datetime
from operator import attrgetter
from pkg_resources import resource_string
from storm.exceptions import DatabaseError
......@@ -44,6 +47,7 @@ from mailman.interfaces.mailinglist import IAcceptableAliasSet
from mailman.interfaces.nntp import NewsgroupModeration
from mailman.interfaces.subscriptions import ISubscriptionService
from mailman.model.bans import Ban
from mailman.model.bounce import BounceContext, BounceEvent
from mailman.testing.helpers import temporary_db
from mailman.testing.layers import ConfigLayer
......@@ -426,3 +430,49 @@ class TestMigration20121015MigratedData(MigrationTestBase):
self.assertEqual(bans[0].list_id, 'test.example.com')
self.assertEqual(bans[1].email, '[email protected]')
self.assertEqual(bans[1].list_id, None)
class TestMigration20130406Schema(MigrationTestBase):
"""Test column migrations."""
def test_pre_upgrade_column_migrations(self):
self._missing_present('bounceevent',
['20130405999999'],
('list_id',),
('list_name',))
def test_post_upgrade_column_migrations(self):
self._missing_present('bounceevent',
['20130405999999',
'20130406000000'],
('list_name',),
('list_id',))
class TestMigration20130406MigratedData(MigrationTestBase):
"""Test migrated data."""
def test_migration_bounceevent(self):
# Load all migrations to just before the one we're testing.
self._database.load_migrations('20130405999999')
# Insert a bounce event.
self._database.store.execute("""
INSERT INTO bounceevent VALUES (
1, '[email protected]', '[email protected]',
'2013-04-06 21:12:00', '<[email protected]>',
1, 0);
""")
# Update to the current migration we're testing
self._database.load_migrations('20130406000000')
# The bounce event should exist, but with a list-id instead of a fqdn
# list name.
events = list(self._database.store.find(BounceEvent))
self.assertEqual(len(events), 1)
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].email, '[email protected]')
self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
self.assertEqual(events[0].message_id, '<[email protected]>')
self.assertEqual(events[0].context, BounceContext[1])
self.assertFalse(events[0].processed)
......@@ -60,8 +60,8 @@ class UnrecognizedBounceDisposition(Enum):
class IBounceEvent(Interface):
"""Registration record for a single bounce event."""
list_name = Attribute(
"""The name of the mailing list that received this bounce.""")
list_id = Attribute(
"""The List-ID of the mailing list that received this bounce.""")
email = Attribute(
"""The email address that bounced.""")
......
......@@ -43,15 +43,15 @@ class BounceEvent(Model):
"""See `IBounceEvent`."""
id = Int(primary=True)
list_name = Unicode()
list_id = Unicode()
email = Unicode()
timestamp = DateTime()
message_id = Unicode()
context = Enum(BounceContext)
processed = Bool()
def __init__(self, list_name, email, msg, context=None):
self.list_name = list_name
def __init__(self, list_id, email, msg, context=None):
self.list_id = list_id
self.email = email
self.timestamp = now()
self.message_id = msg['message-id']
......@@ -67,7 +67,7 @@ class BounceProcessor:
@dbconnection
def register(self, store, mlist, email, msg, where=None):
"""See `IBounceProcessor`."""
event = BounceEvent(mlist.fqdn_listname, email, msg, where)
event = BounceEvent(mlist.list_id, email, msg, where)
store.add(event)
return event
......
......@@ -39,8 +39,8 @@ of bouncing email addresses. These are passed one-by-one to the registration
interface.
>>> event = processor.register(mlist, '[email protected]', msg)
>>> print event.list_name
test@example.com
>>> print event.list_id
test.example.com
>>> print event.email
[email protected]
>>> print event.message_id
......
......@@ -58,7 +58,7 @@ Message-Id: <first>
events = list(self._processor.events)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event.list_name, '[email protected]example.com')
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '[email protected]')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
......@@ -68,7 +68,7 @@ Message-Id: <first>
unprocessed = list(self._processor.unprocessed)
self.assertEqual(len(unprocessed), 1)
event = unprocessed[0]
self.assertEqual(event.list_name, '[email protected]example.com')
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '[email protected]')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
......
......@@ -96,7 +96,7 @@ Message-Id: <first>
events = list(self._processor.events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, '[email protected]')
self.assertEqual(events[0].list_name, '[email protected]example.com')
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].message_id, '<first>')
self.assertEqual(events[0].context, BounceContext.normal)
self.assertEqual(events[0].processed, False)
......@@ -145,7 +145,7 @@ Message-Id: <second>
events = list(self._processor.events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, '[email protected]')
self.assertEqual(events[0].list_name, '[email protected]example.com')
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].message_id, '<second>')
self.assertEqual(events[0].context, BounceContext.probe)
self.assertEqual(events[0].processed, False)
......@@ -175,7 +175,7 @@ Original-Recipient: rfc822; [email protected]
events = list(self._processor.events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, '[email protected]')
self.assertEqual(events[0].list_name, '[email protected]example.com')
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].message_id, '<first>')
self.assertEqual(events[0].context, BounceContext.normal)
self.assertEqual(events[0].processed, False)
......
......@@ -374,7 +374,7 @@ Message-Id: <first>
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event.list_name, '[email protected]example.com')
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email,