Commit faa56a17 authored by Barry Warsaw's avatar Barry Warsaw

* Schema migrations have been implemented.

parent 125ba2db
......@@ -2,7 +2,7 @@
Mailman - The GNU Mailing List Management System
================================================
Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
This is GNU Mailman, a mailing list management system distributed under the
terms of the GNU General Public License (GPL) version 3 or later. The name of
......
......@@ -41,7 +41,7 @@ master_doc = 'README'
# General information about the project.
project = u'GNU Mailman'
copyright = u'1998-2011 by the Free Software Foundation, Inc.'
copyright = u'1998-2012 by the Free Software Foundation, Inc.'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
......
......@@ -44,7 +44,7 @@ def main():
parser = argparse.ArgumentParser(
description=_("""\
The GNU Mailman mailing list management system
Copyright 1998-2011 by the Free Software Foundation, Inc.
Copyright 1998-2012 by the Free Software Foundation, Inc.
http://www.list.org
"""),
formatter_class=argparse.RawDescriptionHelpFormatter)
......
......@@ -189,6 +189,9 @@ class: mailman.database.sqlite.SQLiteDatabase
url: sqlite:///$DATA_DIR/mailman.db
debug: no
# The module path to the migrations modules.
migrations_path: mailman.database.schema
[logging.template]
# This defines various log settings. The options available are:
#
......
......@@ -24,18 +24,18 @@ __all__ = [
import os
import sys
import logging
from flufl.lock import Lock
from lazr.config import as_boolean
from pkg_resources import resource_listdir, resource_string
from storm.cache import GenerationalCache
from storm.locals import create_database, Store
from zope.interface import implements
import mailman.version
from mailman.config import config
from mailman.interfaces.database import IDatabase, SchemaVersionMismatchError
from mailman.interfaces.database import IDatabase
from mailman.model.version import Version
from mailman.utilities.string import expand
......@@ -51,6 +51,10 @@ class StormBaseDatabase:
Use this as a base class for your DB-specific derived classes.
"""
# Tag used to distinguish the database being used. Override this in base
# classes.
TAG = ''
implements(IDatabase)
def __init__(self):
......@@ -87,15 +91,6 @@ class StormBaseDatabase:
"""
raise NotImplementedError
def _get_schema(self):
"""Return the database schema as a string.
This will be loaded into the database when it is first created.
Base classes *must* override this.
"""
raise NotImplementedError
def _pre_reset(self, store):
"""Clean up method for testing.
......@@ -147,34 +142,80 @@ class StormBaseDatabase:
store = Store(database, GenerationalCache())
database.DEBUG = (as_boolean(config.database.debug)
if debug is None else debug)
# Check the master / schema database to see if the version table
# exists. If so, then we assume the database schema is correctly
# initialized. Storm does not currently provide schema creation.
if not self._database_exists(store):
# Initialize the database. Start by getting the schema and
# discarding all blank and comment lines.
lines = self._get_schema().splitlines()
lines = (line for line in lines
self.store = store
self.load_migrations()
store.commit()
def load_migrations(self):
"""Load all not-yet loaded migrations."""
migrations_path = config.database.migrations_path
if '.' in migrations_path:
parent, dot, child = migrations_path.rpartition('.')
else:
parent = migrations_path
child =''
# If the database does not yet exist, load the base schema.
filenames = sorted(resource_listdir(parent, child))
# Find out which schema migrations have already been loaded.
if self._database_exists(self.store):
versions = set(version.version for version in
self.store.find(Version, component='schema'))
else:
versions = set()
for filename in filenames:
module_fn, extension = os.path.splitext(filename)
if extension != '.py':
continue
parts = module_fn.split('_')
if len(parts) < 2:
continue
version = parts[1]
if version in versions:
# This one is already loaded.
continue
module_path = migrations_path + '.' + module_fn
__import__(module_path)
upgrade = getattr(sys.modules[module_path], 'upgrade', None)
if upgrade is None:
continue
upgrade(self, self.store, version, module_path)
def load_schema(self, store, version, filename, module_path):
"""Load the schema from a file.
This is a helper method for migration classes to call.
:param store: The Storm store to load the schema into.
:type store: storm.locals.Store`
:param version: The schema version identifier of the form
YYYYMMDDHHMMSS.
:type version: string
:param filename: The file name containing the schema to load. Pass
`None` if there is no schema file to load.
:type filename: string
:param module_path: The fully qualified Python module path to the
migration module being loaded. This is used to record information
for use by the test suite.
:type module_path: string
"""
if filename is not None:
contents = resource_string('mailman.database.schema', filename)
# Discard all blank and comment lines.
lines = (line for line in contents.splitlines()
if line.strip() != '' and line.strip()[:2] != '--')
sql = NL.join(lines)
for statement in sql.split(';'):
if statement.strip() != '':
store.execute(statement + ';')
# Validate schema version.
v = store.find(Version, component='schema').one()
if not v:
# Database has not yet been initialized
v = Version(component='schema',
version=mailman.version.DATABASE_SCHEMA_VERSION)
store.add(v)
elif v.version <> mailman.version.DATABASE_SCHEMA_VERSION:
# XXX Update schema
raise SchemaVersionMismatchError(v.version)
self.store = store
store.commit()
# Add a marker that indicates the migration version being applied.
store.add(Version(component='schema', version=version))
# Add a marker so that the module name can be found later. This is
# used by the test suite to reset the database between tests.
store.add(Version(component=version, version=module_path))
def _reset(self):
"""See `IDatabase`."""
from mailman.database.model import ModelMeta
self.store.rollback()
ModelMeta._reset(self.store)
self.store.commit()
=================
Schema migrations
=================
The SQL database schema will over time require upgrading to support new
features. This is supported via schema migration.
Migrations are embodied in individual Python classes, which themselves may
load SQL into the database. The naming scheme for migration files is:
s_YYYYMMDDHHMMSS_comment.py
where `YYYYMMDDHHMMSS` is a required numeric year, month, day, hour, minute,
and second specifier providing unique ordering for processing. Only this
component of the file name is used to determine the ordering. (The `s_`
prefix is required due to Python module naming requirements).
The optional `comment` part of the file name can be used as a short
description for the migration, although comments and docstrings in the
migration files should be used for more detailed descriptions.
Migrations are applied automatically when Mailman starts up, but can also be
applied at any time by calling in the API directly. Once applied, a
migration's version string is registered so it will not be applied again.
We see that the base migration is already applied.
>>> from mailman.model.version import Version
>>> results = config.db.store.find(Version, component='schema')
>>> results.count()
1
>>> base = results.one()
>>> print base.component
schema
>>> print base.version
00000000000000
Migrations
==========
Migrations can be loaded at any time, and can be found in the migrations path
specified in the configuration file.
.. Create a temporary directory for the migrations::
>>> import os, sys, tempfile
>>> tempdir = tempfile.mkdtemp()
>>> path = os.path.join(tempdir, 'migrations')
>>> os.makedirs(path)
>>> sys.path.append(tempdir)
>>> config.push('migrations', """
... [database]
... migrations_path: migrations
... """)
Here is an example migrations module. The key part of this interface is the
``upgrade()`` method, which takes four arguments:
* `database` - The database class, as derived from `StormBaseDatabase`
* `store` - The Storm `Store` object.
* `version` - The version string as derived from the migrations module's file
name. This will include only the `YYYYMMDDHHMMSS` string.
* `module_path` - The dotted module path to the migrations module, suitable
for lookup in `sys.modules`.
This migration module just adds a marker to the `version` table.
>>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
... pass
>>> with open(os.path.join(path, 's_20120211000000.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
... def upgrade(database, store, version, module_path):
... v = Version(component='test', version=version)
... store.add(v)
... database.load_schema(store, version, None, module_path)
... """
This will load the new migration, since it hasn't been loaded before.
>>> config.db.load_migrations()
>>> results = config.db.store.find(Version, component='schema')
>>> for result in sorted(result.version for result in results):
... print result
00000000000000
20120211000000
>>> test = config.db.store.find(Version, component='test').one()
>>> print test.version
20120211000000
Migrations will only be loaded once.
>>> with open(os.path.join(path, 's_20120211000001.py'), 'w') as fp:
... print >> fp, """
... from __future__ import unicode_literals
... from mailman.model.version import Version
... _marker = 801
... def upgrade(database, store, version, module_path):
... global _marker
... # Pad enough zeros on the left to reach 14 characters wide.
... marker = '{0:=#014d}'.format(_marker)
... _marker += 1
... v = Version(component='test', version=marker)
... store.add(v)
... database.load_schema(store, version, None, module_path)
... """
The first time we load this new migration, we'll get the 801 marker.
>>> config.db.load_migrations()
>>> results = config.db.store.find(Version, component='schema')
>>> for result in sorted(result.version for result in results):
... print result
00000000000000
20120211000000
20120211000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20120211000000
We do not get an 802 marker because the migration has already been loaded.
>>> config.db.load_migrations()
>>> results = config.db.store.find(Version, component='schema')
>>> for result in sorted(result.version for result in results):
... print result
00000000000000
20120211000000
20120211000001
>>> test = config.db.store.find(Version, component='test')
>>> for marker in sorted(marker.version for marker in test):
... print marker
00000000000801
20120211000000
.. Clean up the temporary directory::
>>> config.pop('migrations')
>>> sys.path.remove(tempdir)
>>> import shutil
>>> shutil.rmtree(tempdir)
......@@ -25,6 +25,8 @@ __all__ = [
]
import sys
from operator import attrgetter
from storm.properties import PropertyPublisherMeta
......@@ -50,12 +52,37 @@ class ModelMeta(PropertyPublisherMeta):
@staticmethod
def _reset(store):
from mailman.config import config
from mailman.model.version import Version
config.db._pre_reset(store)
# Give each schema migration a chance to do its pre-reset. See below
# for calling its post reset too.
versions = sorted(version.version for version in
store.find(Version, component='schema'))
migrations = {}
for version in versions:
# We have to give the migrations module that loaded this version a
# chance to do both pre- and post-reset operations. The following
# find the actual the module path for the migration. See
# StormBaseDatabase.load_schema().
migration = store.find(Version, component=version).one()
if migration is None:
continue
migrations[version] = module_path = migration.version
module = sys.modules[module_path]
pre_reset = getattr(module, 'pre_reset', None)
if pre_reset is not None:
pre_reset(store)
# Make sure this is deterministic, by sorting on the storm table name.
classes = sorted(ModelMeta._class_registry,
key=attrgetter('__storm_table__'))
for model_class in classes:
store.find(model_class).remove()
# Now give each migration a chance to do post-reset operations.
for version in versions:
module = sys.modules[migrations[version]]
post_reset = getattr(module, 'post_reset', None)
if post_reset is not None:
post_reset(store)
config.db._post_reset(store)
......
......@@ -26,7 +26,6 @@ __all__ = [
from operator import attrgetter
from pkg_resources import resource_string
from mailman.database.base import StormBaseDatabase
......@@ -35,6 +34,8 @@ from mailman.database.base import StormBaseDatabase
class PostgreSQLDatabase(StormBaseDatabase):
"""Database class for PostgreSQL."""
TAG = 'postgres'
def _database_exists(self, store):
"""See `BaseDatabase`."""
table_query = ('SELECT table_name FROM information_schema.tables '
......@@ -43,16 +44,13 @@ class PostgreSQLDatabase(StormBaseDatabase):
store.execute(table_query))
return 'version' in table_names
def _get_schema(self):
"""See `BaseDatabase`."""
return resource_string('mailman.database.sql', 'postgres.sql')
def _post_reset(self, store):
"""PostgreSQL-specific test suite cleanup.
Reset the <tablename>_id_seq.last_value so that primary key ids
restart from zero for new tests.
"""
super(PostgreSQLDatabase, self)._post_reset(store)
from mailman.database.model import ModelMeta
classes = sorted(ModelMeta._class_registry,
key=attrgetter('__storm_table__'))
......
......@@ -319,7 +319,7 @@ CREATE TABLE pendedkeyvalue (
CREATE TABLE version (
id SERIAL NOT NULL,
component TEXT,
version INTEGER,
version TEXT,
PRIMARY KEY (id)
);
......
# Copyright (C) 2012 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""Load the base schema."""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'upgrade',
'post_reset',
'pre_reset',
]
_migration_path = None
VERSION = '00000000000000'
def upgrade(database, store, version, module_path):
filename = '{0}.sql'.format(database.TAG)
database.load_schema(store, version, filename, module_path)
def pre_reset(store):
global _migration_path
# Save the entry in the Version table for the test suite reset. This will
# be restored below.
from mailman.model.version import Version
result = store.find(Version, component=VERSION).one()
# Yes, we abuse this field.
_migration_path = result.version
def post_reset(store):
from mailman.model.version import Version
# We need to preserve the Version table entry for this migration, since
# its existence defines the fact that the tables have been loaded.
store.add(Version(component='schema', version=VERSION))
store.add(Version(component=VERSION, version=_migration_path))
......@@ -295,7 +295,7 @@ CREATE INDEX ix_user_user_id ON user (_user_id);
CREATE TABLE version (
id INTEGER NOT NULL,
component TEXT,
version INTEGER,
version TEXT,
PRIMARY KEY (id)
);
......
......@@ -27,7 +27,6 @@ __all__ = [
import os
from pkg_resources import resource_string
from urlparse import urlparse
from mailman.database.base import StormBaseDatabase
......@@ -37,6 +36,8 @@ from mailman.database.base import StormBaseDatabase
class SQLiteDatabase(StormBaseDatabase):
"""Database class for SQLite."""
TAG = 'sqlite'
def _database_exists(self, store):
"""See `BaseDatabase`."""
table_query = 'select tbl_name from sqlite_master;'
......@@ -53,7 +54,3 @@ class SQLiteDatabase(StormBaseDatabase):
# Ignore errors
if fd > 0:
os.close(fd)
def _get_schema(self):
"""See `BaseDatabase`."""
return resource_string('mailman.database.sql', 'sqlite.sql')
......@@ -4,7 +4,7 @@
GNU Mailman Acknowledgments
===========================
Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
Core Developers
......
......@@ -18,7 +18,7 @@ Learn more about GNU Mailman in the `Getting Started`_ documentation.
Copyright
=========
Copyright 1998-2011 by the Free Software Foundation, Inc.
Copyright 1998-2012 by the Free Software Foundation, Inc.
This file is part of GNU Mailman.
......
......@@ -2,7 +2,7 @@
Mailman - The GNU Mailing List Management System
================================================
Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
Here is a history of user visible changes to Mailman.
......@@ -14,6 +14,7 @@ Here is a history of user visible changes to Mailman.
Architecture
------------
* Schema migrations have been implemented.
* Implement the style manager as a utility instead of an attribute hanging
off the `mailman.config.config` object.
* PostgreSQL support contributed by Stephen A. Goss. (LP: #860159)
......
......@@ -23,14 +23,12 @@ __metaclass__ = type
__all__ = [
'DatabaseError',
'IDatabase',
'SchemaVersionMismatchError',
]
from zope.interface import Interface
from mailman.interfaces.errors import MailmanError
from mailman.version import DATABASE_SCHEMA_VERSION
......@@ -38,19 +36,6 @@ class DatabaseError(MailmanError):
"""A problem with the database occurred."""
class SchemaVersionMismatchError(DatabaseError):
"""The database schema version number did not match what was expected."""
def __init__(self, got):
super(SchemaVersionMismatchError, self).__init__()
self._got = got
def __str__(self):
return ('Incompatible database schema version '
'(got: {0}, expected: {1})'.format(
self._got, DATABASE_SCHEMA_VERSION))
class IDatabase(Interface):
"""Database layer interface."""
......
......@@ -32,7 +32,7 @@ from mailman.database.model import Model
class Version(Model):
id = Int(primary=True)
component = Unicode()
version = Int()
version = Unicode()
def __init__(self, component, version):
super(Version, self).__init__()
......
......@@ -40,9 +40,6 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
(REL_LEVEL << 4) | (REL_SERIAL << 0))
# SQL database schema version
DATABASE_SCHEMA_VERSION = 1
# qfile/*.db schema version number
QFILE_SCHEMA_VERSION = 3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment