Commit 2d904f6e authored by Mike Ryan's avatar Mike Ryan

#67: Add smoke test.

parent 1609a858
Pipeline #54671437 passed with stage
in 2 minutes and 28 seconds
......@@ -39,10 +39,10 @@ arraytosql:
num: 38
related: 1
transform:
# @todo: have 'configuration' and 'property' keys. Configuration option:
# automatically copy source record to data record so only changed/new
# destination properties need to be mapped.
#uniqueid: id
# @todo: have 'configuration' and 'property' keys.
# Configuration option: automatically copy source record to data record so
# only changed/new destination properties need to be mapped.
# uniqueid: id
foo:
-
class: Soong\Transformer\Copy
......
DROP TABLE IF EXISTS `beer`;
CREATE TABLE `beer` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`body` text DEFAULT NULL,
`excerpt` varchar(255) DEFAULT NULL,
`userid` int(11) DEFAULT NULL,
`userid` int DEFAULT NULL,
`image_path` varchar(255) DEFAULT NULL,
`image_alt` varchar(255) DEFAULT NULL,
`image_title` varchar(255) DEFAULT NULL,
`image_description` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
);
DROP TABLE IF EXISTS `beer_comments`;
CREATE TABLE `beer_comments` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`beerid` int(11) DEFAULT NULL,
`parent` int(11) DEFAULT NULL,
`subject` varchar(255) DEFAULT NULL,
`body` text DEFAULT NULL,
`userid` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
);
DROP TABLE IF EXISTS `beer_terms`;
CREATE TABLE `beer_terms` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`id` int NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL,
`description` varchar(255) DEFAULT NULL,
`parent` varchar(255) DEFAULT NULL,
......@@ -32,16 +22,10 @@ CREATE TABLE `beer_terms` (
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`)
);
DROP TABLE IF EXISTS `beer_userfaves`;
CREATE TABLE `beer_userfaves` (
`beerid` int(11) NOT NULL,
`userid` int(11) NOT NULL,
PRIMARY KEY (`userid`,`beerid`)
);
DROP TABLE IF EXISTS `beer_users`;
CREATE TABLE `beer_users` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`published` int(11) DEFAULT NULL,
`id` int NOT NULL AUTO_INCREMENT,
`published` int DEFAULT NULL,
`registration_date` varchar(20) DEFAULT NULL,
`username` varchar(255) DEFAULT NULL,
`display_name` varchar(255) DEFAULT NULL,
......
DROP TABLE IF EXISTS `beer`;
CREATE TABLE `beer` (
`id` integer PRIMARY KEY,
`name` varchar(255) DEFAULT NULL,
`body` text DEFAULT NULL,
`excerpt` varchar(255) DEFAULT NULL,
`userid` int DEFAULT NULL,
`image_path` varchar(255) DEFAULT NULL,
`image_alt` varchar(255) DEFAULT NULL,
`image_title` varchar(255) DEFAULT NULL,
`image_description` varchar(255) DEFAULT NULL
);
DROP TABLE IF EXISTS `beer_terms`;
CREATE TABLE `beer_terms` (
`id` integer PRIMARY KEY,
`name` varchar(255) NOT NULL,
`description` varchar(255) DEFAULT NULL,
`parent` varchar(255) DEFAULT NULL,
`region` varchar(255) DEFAULT NULL,
`hoppiness` varchar(255) DEFAULT NULL,
UNIQUE(`name`)
);
DROP TABLE IF EXISTS `beer_users`;
CREATE TABLE `beer_users` (
`id` integer PRIMARY KEY,
`published` int DEFAULT NULL,
`registration_date` varchar(20) DEFAULT NULL,
`username` varchar(255) DEFAULT NULL,
`display_name` varchar(255) DEFAULT NULL,
`pass` varchar(255) DEFAULT NULL,
`email` varchar(255) NOT NULL,
`taster` varchar(255) DEFAULT NULL
);
DROP TABLE IF EXISTS `extractsource`;
CREATE TABLE `extractsource` (
`uniqueid` int(11) NOT NULL AUTO_INCREMENT,
`uniqueid` int NOT NULL AUTO_INCREMENT,
`foo` varchar(255) NOT NULL,
`bar` varchar(255) NOT NULL,
`num` int(11) NOT NULL,
`related` int(11) DEFAULT NULL,
`num` int NOT NULL,
`related` int DEFAULT NULL,
PRIMARY KEY (`uniqueid`)
);
DROP TABLE IF EXISTS `extractsource`;
CREATE TABLE `extractsource` (
`uniqueid` integer PRIMARY KEY,
`foo` varchar(255) NOT NULL,
`bar` varchar(255) NOT NULL,
`num` int NOT NULL,
`related` int DEFAULT NULL
);
......@@ -17,6 +17,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- `TaskPipeline` component for managing groups of tasks.
- `ComponentNotFound` and `DuplicateTask` exceptions added.
- Tests for `Extractor`, `KeyMap`, `Loader`, and `Task` components.
- Smoke test to make sure all provided examples keep working.
### Removed
- `isCompleted` method on `Task` - unneeded until we add dependencies.
......
site_name: Soong ETL
theme: readthedocs
nav:
- Introduction:
- index.md
- example.md
- 'API Reference': api/html/index.html
- 'Contributing': CONTRIBUTING.md
- 'Code of Conduct': CODE_OF_CONDUCT.md
- 'Change log': CHANGELOG.md
site_url: http://soongetl.org
repo_url: https://gitlab.com/soongetl/soong
markdown_extensions:
- toc:
permalink: true
- admonition
- markdown_include.include:
base_path: docs
......@@ -66,5 +66,7 @@ class DBAL extends LoaderBase
$counter++;
}
$queryBuilder->execute();
$this->connection->close();
$this->connection = null;
}
}
......@@ -170,6 +170,9 @@ class EtlTask extends Task implements EtlTaskInterface
$loader = $this->getLoader();
$keyMap = $this->getKeyMap();
if (empty($keyMap)) {
return;
}
foreach ($keyMap->iterate() as $extractedKey) {
$loadedKey = $keyMap->lookupLoadedKey($extractedKey);
......
......@@ -49,7 +49,7 @@ trait DBALTesting
}
/**
* @inheritdoc
* inheritdoc
*/
protected function tearDown() : void
{
......
<?php
namespace Soong\Tests;
use Doctrine\DBAL\DriverManager;
use Doctrine\DBAL\FetchMode;
use PHPUnit\Framework\TestCase;
use Soong\Tests\DBAL\DBALTesting;
use Symfony\Component\Yaml\Yaml;
/**
* Run the provided examples and make sure they work.
*/
class SmokeTest extends TestCase
{
use DBALTesting {
tearDown as tearDownDBAL;
}
/**
* Temporary directory for our munged YAML file.
*
* @var string
*/
protected $configDirectory = '';
/**
* Test the migrations defined in config/etl.yml.
*/
public function testEtl() : void
{
$this->populateDatabase(['data/extractsource.sqlite.sql']);
$yamlDir = $this->rewriteYaml(['config/etl.yml']);
// Migrating arraytosql - populates extractsource table and map table.
$output = [];
$expectedOutput = ['Executing arraytosql'];
exec("bin/soong migrate arraytosql --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
$this->validateTableContents('extractsource', 'uniqueid', [
[
'uniqueid' => 1,
'foo' => 'first record',
'bar' => 'description of first record',
'num' => 1,
'related' => null,
],
[
'uniqueid' => 2,
'foo' => 'second record',
'bar' => 'description of second record',
'num' => 2,
'related' => null,
],
[
'uniqueid' => 3,
'foo' => 'third record',
'bar' => 'description of third record',
'num' => 38,
'related' => 1,
],
]);
$this->validateTableContents('map_arraytosql', 'extracted_key_1', [
[
'extracted_key_1' => 1,
'loaded_key_1' => 1,
],
[
'extracted_key_1' => 5,
'loaded_key_1' => 2,
],
[
'extracted_key_1' => 8,
'loaded_key_1' => 3,
],
]);
// Migrating sqltocsv - outputs CSV.
$output = [];
$expectedOutput = [
'Executing sqltocsv',
'foo,bar,num,relation,id',
'first record,Description of first record,4,,0',
'second record,Description of second record,6,,1',
'third record,Description of third record,78,1,2',
];
exec("bin/soong migrate sqltocsv --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
// Rolling back sqltocsv - does nothing.
$output = [];
$expectedOutput = [
'Executing sqltocsv',
];
exec("bin/soong rollback sqltocsv --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
// Rolling back arraytosql - empties tables.
// @todo: Restore when https://gitlab.com/soongetl/soong/issues/69 is
// fixed.
/* $output = [];
$expectedOutput = [
'Executing arraytosql',
];
exec("bin/soong rollback arraytosql --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
$this->validateTableContents('extractsource', 'uniqueid', []);
$this->validateTableContents('map_arraytosql', 'extracted_key_1', []);*/
}
/**
* Test the migrations defined in config/beer*.yml.
*/
public function testBeer() : void
{
$this->populateDatabase(['data/beer.sqlite.sql']);
$yamlDir = $this->rewriteYaml([
'config/beeraccounts.yml',
'config/beercontent.yml',
'config/beertopics.yml',
]);
// Migrating beeraccounts - populates beer_users table and map table.
$output = [];
$expectedOutput = ['Executing beeraccounts'];
exec("bin/soong migrate beeraccounts --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
$this->validateTableContents('beer_users', 'id', [
[
'id' => 1,
'published' => 1,
'registration_date' => '2010-03-30 10:31:05',
'username' => 'alice',
'display_name' => 'alice in beerland',
'pass' => 'alicepass',
'email' => '[email protected]',
'taster' => 'Professional Taster',
],
[
'id' => 2,
'published' => 1,
'registration_date' => '2010-04-04 10:31:05',
'username' => 'alice',
'display_name' => 'alice in aleland',
'pass' => 'alicepass',
'email' => '[email protected]',
'taster' => 'Consumer',
],
[
'id' => 3,
'published' => 0,
'registration_date' => '2007-03-15 10:31:05',
'username' => 'bob',
'display_name' => 'rebob',
'pass' => 'bobpass',
'email' => '[email protected]',
'taster' => 'Professional Taster',
],
[
'id' => 4,
'published' => 1,
'registration_date' => '2004-02-29 10:31:05',
'username' => 'charlie',
'display_name' => 'charlie chocolate',
'pass' => 'mykids',
'email' => '[email protected]',
'taster' => 'Consumer',
],
]);
$this->validateTableContents('map_beeraccounts', 'extracted_key_1', [
[
'extracted_key_1' => 1,
'loaded_key_1' => 1,
],
[
'extracted_key_1' => 2,
'loaded_key_1' => 2,
],
[
'extracted_key_1' => 3,
'loaded_key_1' => 3,
],
[
'extracted_key_1' => 4,
'loaded_key_1' => 4,
],
]);
// Migrating beertopics - populates beer_terms table and map table.
$output = [];
$expectedOutput = ['Executing beertopics'];
exec("bin/soong migrate beertopics --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
$this->validateTableContents('beer_terms', 'id', [
[
'id' => 1,
'name' => 'ale',
'description' => 'traditional',
'parent' => null,
'region' => 'Medieval British Isles',
'hoppiness' => 'Medium',
],
[
'id' => 2,
'name' => 'pilsner',
'description' => 'refreshing',
'parent' => null,
'region' => 'Pilsen, Bohemia (now Czech Republic)',
'hoppiness' => 'Low',
],
[
'id' => 3,
'name' => 'red ale',
'description' => 'colorful',
'parent' => 1,
'region' => '',
'hoppiness' => '',
],
]);
$this->validateTableContents('map_beertopics', 'extracted_key_1', [
[
'extracted_key_1' => 'ale',
'loaded_key_1' => 1,
],
[
'extracted_key_1' => 'pilsner',
'loaded_key_1' => 2,
],
[
'extracted_key_1' => 'red ale',
'loaded_key_1' => 3,
],
]);
// Migrating beercontent - populates beer table and map table.
$output = [];
$expectedOutput = ['Executing beercontent'];
exec("bin/soong migrate beercontent --directory=$yamlDir", $output);
$this->assertEquals($expectedOutput, $output);
$this->validateTableContents('beer', 'id', [
[
'id' => 1,
'name' => 'Boddington',
'body' => 'English occasionally get something right',
'excerpt' => 'A treat',
'userid' => 1,
'image_path' => '',
'image_alt' => '',
'image_title' => '',
'image_description' => '',
],
[
'id' => 2,
'name' => 'Miller Lite',
'body' => 'We love Miller Brewing',
'excerpt' => 'Tasteless',
'userid' => 3,
'image_path' => '',
'image_alt' => '',
'image_title' => '',
'image_description' => '',
],
[
'id' => 3,
'name' => 'Heineken',
'body' => 'Blab Blah Blah Green',
'excerpt' => 'Green',
'userid' => null,
'image_path' => 'heineken.jpg',
'image_alt' => 'Heinekin alt',
'image_title' => 'Heinekin title',
'image_description' => 'Heinekin description',
],
]);
$this->validateTableContents('map_beercontent', 'extracted_key_1', [
[
'extracted_key_1' => 99999997,
'loaded_key_1' => 1,
],
[
'extracted_key_1' => 99999998,
'loaded_key_1' => 2,
],
[
'extracted_key_1' => 99999999,
'loaded_key_1' => 3,
],
]);
// @todo: Test rollback when https://gitlab.com/soongetl/soong/issues/69
// is fixed.
}
protected function populateDatabase(array $sqlFiles)
{
$this->dbSetup();
$connection = DriverManager::getConnection($this->configuration['connection']);
foreach ($sqlFiles as $sqlFile) {
$fileContents = file_get_contents($sqlFile);
$sqlStatements = array_filter(explode(';', $fileContents));
foreach ($sqlStatements as $sql) {
// Convert MySQL to SQLite.
$sql = str_replace(
'int NOT NULL AUTO_INCREMENT',
'INTEGER PRIMARY KEY',
trim($sql)
);
if ($sql) {
$connection->executeQuery($sql);
}
}
}
$connection->close();
}
/**
* Inject our test db connection into YAML files to be tested.
*
* @param array $yamlFiles
* List of YAML files to rewrite, relative to project root.
*
* @return string
* The temporary directory holding the rewritten YAML files.
*/
protected function rewriteYaml(array $yamlFiles) : string
{
$this->configDirectory = '/tmp/' . bin2hex(random_bytes(6)) . '/';
mkdir($this->configDirectory);
foreach ($yamlFiles as $yamlFile) {
// Rewrite the connection configuration with our SQLite connection.
$configuration = Yaml::parseFile($yamlFile);
foreach ($configuration as $taskId => $taskInfo) {
foreach ($taskInfo['configuration'] as $componentKey => $componentInfo) {
if (isset($componentInfo['configuration']['connection'])) {
$configuration[$taskId]['configuration'][$componentKey]['configuration']['connection'] =
$this->configuration['connection'];
}
}
}
$configurationString = Yaml::dump($configuration, 6, 4);
file_put_contents(
$this->configDirectory . basename($yamlFile),
$configurationString
);
}
return $this->configDirectory;
}
/**
* Compare the contents of a database table with the expected results.
*
* @param string $tableName
* Table to compare.
* @param string $sortColumn
* Table column to sort by to ensure consistency with expected results.
* @param array $expectedContents
* Each array member is an array representing one table row, keyed by
* column name.
*/
protected function validateTableContents(string $tableName, string $sortColumn, array $expectedContents) : void
{
$connection = DriverManager::getConnection($this->configuration['connection']);
$statement = $connection->executeQuery("SELECT * FROM $tableName ORDER BY $sortColumn");
$actualContents = [];
while ($row = $statement->fetch(FetchMode::ASSOCIATIVE)) {
// Ignore map table hash columns.
unset($row['hash']);
$actualContents[] = $row;
}
$connection->close();
$this->assertEquals($expectedContents, $actualContents);
}
/**
* inheritdoc
*/
protected function tearDown() : void
{
// Remove our munged YAML and its directory.
array_map('unlink', glob($this->configDirectory . '*.yml'));
rmdir(rtrim($this->configDirectory, '/'));
$this->tearDownDBAL();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment