Commit 7285f367 authored by Mike Ryan's avatar Mike Ryan

#14: Use Config library for reading configuration in various formats.

parent 2d904f6e
Pipeline #55073782 passed with stage
in 2 minutes and 40 seconds
......@@ -21,6 +21,7 @@
"require": {
"php": ">=7.1",
"doctrine/dbal": "^2.7",
"hassankhan/config": "^2.0",
"league/csv": "^9.1",
"monolog/monolog": "^1.23",
"symfony/console": "^4.1",
......
{
"sqltocsv_json": {
"class": "Soong\\Task\\EtlTask",
"configuration": {
"record_class": "Soong\\Data\\Record",
"extract": {
"class": "Soong\\Extractor\\DBAL",
"configuration": {
"data_record_class": "Soong\\Data\\Record",
"connection": {
"dbname": "etltemp",
"user": "root",
"host": "127.0.0.1",
"port": 3306,
"driver": "pdo_mysql"
},
"query": "SELECT * FROM extractsource",
"key_properties": {
"uniqueid": {
"type": "integer"
}
}
}
},
"transform": {
"foo": "foo",
"bar": [
{
"class": "Soong\\Transformer\\UcFirst",
"source": "bar"
}
],
"num": [
{
"class": "Soong\\Transformer\\Increment",
"source": "num"
},
{
"class": "Soong\\Transformer\\Double"
}
],
"relation": "related"
},
"load": {
"class": "Soong\\Loader\\Csv",
"configuration": {
"key_properties": {
"id": {
"type": "integer"
}
},
"properties": [
"foo",
"bar",
"num",
"relation",
"id"
]
}
}
}
}
}
<?xml version="1.0"?>
<task>
<sqltocsv_xml>
<class>Soong\Task\EtlTask</class>
<configuration>
<record_class>Soong\Data\Record</record_class>
<!-- Note we have no key_map - we are using this migration as an exporter. -->
<extract>
<class>Soong\Extractor\DBAL</class>
<configuration>
<data_record_class>Soong\Data\Record</data_record_class>
<connection>
<!-- Replace with your test database credentials. -->
<dbname>etltemp</dbname>
<user>root</user>
<host>127.0.0.1</host>
<port>3306</port>
<driver>pdo_mysql</driver>
</connection>
<query>SELECT * FROM extractsource</query>
<key_properties>
<uniqueid>
<type>integer</type>
</uniqueid>
</key_properties>
</configuration>
</extract>
<transform>
<foo>foo</foo>
<bar>
<transformer>
<class>Soong\Transformer\UcFirst</class>
<source>bar</source>
</transformer>
</bar>
<num>
<!-- Without deduping elements, parser introduces extra layer. -->
<transformer_1>
<class>Soong\Transformer\Increment</class>
<source>num</source>
</transformer_1>
<transformer_2>
<class>Soong\Transformer\Double</class>
</transformer_2>
</num>
<relation>related</relation>
</transform>
<load>
<class>Soong\Loader\Csv</class>
<configuration>
<key_properties>
<id>
<type>integer</type>
</id>
</key_properties>
<properties>
<!-- Without deduping elements, parser introduces extra layer. -->
<property_name_1>foo</property_name_1>
<property_name_2>bar</property_name_2>
<property_name_3>num</property_name_3>
<property_name_4>relation</property_name_4>
<property_name_5>id</property_name_5>
</properties>
</configuration>
</load>
</configuration>
</sqltocsv_xml>
</task>
......@@ -14,6 +14,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- Static methods removed from `Task` component and moved to non-static methods on the new `TaskPipeline` component: `addTask()`, `getTask()`, `getAllTasks()`.
### Added
- Commands now use `hassankhan/config` instead of custom YAML handling - configuration now can be YAML, JSON, or XML transparently (examples provided for each).
- `TaskPipeline` component for managing groups of tasks.
- `ComponentNotFound` and `DuplicateTask` exceptions added.
- Tests for `Extractor`, `KeyMap`, `Loader`, and `Task` components.
......
......@@ -3,11 +3,11 @@ declare(strict_types=1);
namespace Soong\Console\Command;
use Noodlehaus\Config;
use Soong\Task\TaskPipeline;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Yaml\Yaml;
/**
* Base class for all Soong console commands.
......@@ -68,24 +68,12 @@ class EtlCommand extends Command
{
$this->pipeline = new TaskPipeline([]);
foreach ($directoryNames as $directoryName) {
$directory = new \RecursiveDirectoryIterator(
$directoryName,
\FilesystemIterator::CURRENT_AS_FILEINFO | \FilesystemIterator::SKIP_DOTS
);
$filter = new YamlRecursiveFilterIterator($directory);
$allFiles = new \RecursiveIteratorIterator(
$filter,
\RecursiveIteratorIterator::SELF_FIRST
);
foreach ($allFiles as $file) {
if (!$file->isDir()) {
$configuration = Yaml::parse(file_get_contents($file->getPathname()));
foreach ($configuration as $id => $taskConfiguration) {
$task = new $taskConfiguration['class']($taskConfiguration['configuration']
+ ['pipeline' => $this->pipeline]);
$this->pipeline->addTask($id, $task);
}
}
$conf = Config::load($directoryName);
foreach ($conf->all() as $id => $taskConfiguration) {
// Inject the pipeline into each task so it knows its parent.
$task = new $taskConfiguration['class']($taskConfiguration['configuration']
+ ['pipeline' => $this->pipeline]);
$this->pipeline->addTask($id, $task);
}
}
}
......
<?php
declare(strict_types=1);
namespace Soong\Console\Command;
/**
* Gather all YAML files in a directory.
*/
class YamlRecursiveFilterIterator extends \RecursiveFilterIterator
{
/**
* @inheritdoc
*/
public function accept()
{
$file = parent::current();
if ($file->isDir()) {
return true;
}
$name = $file->getFilename();
return (substr($name, -4) == '.yml' || substr($name, -5) == '.yaml');
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment