...
 
Commits (3)
<?php
/**
* ProcessHello.info.php
*
* Return information about this module.
*
* If preferred, you can use a getModuleInfo() method in your module file,
* or you can use a ModuleName.info.json file (if you prefer JSON definition).
*
*/
$info = array(
'title' => 'ProcessRockFinder',
'summary' => 'ProcessModule to test RockFinders',
'version' => 3,
'author' => 'Bernhard Baumrock, baumrock.com',
'icon' => 'bolt',
'requires' => ['RockFinder'],
'page' => array(
'name' => 'rockfindertester',
'parent' => 'setup',
'title' => $this->_('RockFinder Tester'),
),
);
$(document).ready(function() {
hljs.initHighlightingOnLoad();
// submit form on ctrl+enter
$('#wrap_Inputfield_code').keydown(function (e) {
if ((e.ctrlKey || e.altKey) && e.keyCode == 13) {
$('#submit').click();
}
});
});
document.addEventListener('RockGridItemBeforeInit', function(e) {
if(e.target.id != 'RockGridItem_ProcessRockFinderResult') return;
var grid = RockGrid.getGrid(e.target.id);
// overwrite rowactions for first column
col = grid.getColDef('id');
col.cellRenderer = function(params) {
var grid = RockGrid.getGrid(params);
// extend the current renderer and add custom icons
return '<span>' + params.data.id + '</span>' + RockGrid.renderers.actionItems(params, [{
icon: 'fa fa-search',
href: '/admin/page/edit/?id=' + params.data.id,
str: 'show',
class: 'class="pw-panel"',
target: 'target="_blank"',
}]);
}
});
document.addEventListener('RockGridButtons.beforeRender', function(e) {
if(e.target.id != 'RockGridWrapper_ProcessRockFinderResult') return;
var grid = RockGrid.getGrid(e.target);
var plugin = grid.plugins.buttons;
// remove a btton
plugin.buttons.remove('refresh');
});
\ No newline at end of file
<?php namespace ProcessWire;
class ProcessRockFinder extends Process {
/**
* init the module
*/
public function init() {
parent::init(); // always remember to call the parent init
}
/**
* execute the tester interface
*/
public function ___execute() {
$form = $this->modules->get('InputfieldForm');
// if reset parameter is set, add comments to tester.txt file
$file = $this->config->paths->assets . 'RockGrid/tester.txt';
if($this->input->get->reset) {
$str = file_get_contents($file);
file_put_contents($file, str_replace("\n", "\n// ", $str));
$this->session->redirect('./');
}
// if tester.txt does not exist create it from sample file
if(!is_file($file)) {
$this->files->copy(__DIR__ . '/exampleTester.php', $file);
}
$f = $this->modules->get('InputfieldTextarea');
if($ace = $this->modules->get('InputfieldAceExtended')) {
$ace->rows = 10;
$ace->theme = 'monokai';
$ace->mode = 'php';
$ace->setAdvancedOptions(array(
'highlightActiveLine' => false,
'showLineNumbers' => false,
'showGutter' => false,
'tabSize' => 2,
'printMarginColumn' => false,
));
$f = $ace;
}
$f->notes = "Execute on CTRL+ENTER or ALT+ENTER";
$f->notes .= "\nThe code must return either an SQL statement or a RockFinder instance";
if(!$ace) $f->notes .= "\nYou can install 'InputfieldAceExtended' for better code editing";
$f->name = 'code';
$f->value = $code = $this->input->post->code ?: file_get_contents($this->config->paths->assets . 'RockGrid/tester.txt');
$f->label = 'Code to execute';
$form->add($f);
// save code to file
file_put_contents($this->config->paths->assets . 'RockGrid/tester.txt', $code);
$search = ['<?php', 'new RockFinder'];
$replace = ['//', 'new \ProcessWire\RockFinder'];
$code = eval(str_replace($search, $replace, $code));
$f = $this->modules->get('InputfieldRockGrid');
if($f) {
$f->type = 'RockGrid';
$f->label = 'Result';
$f->name = 'ProcessRockFinderResult';
$f->debug = true;
if($code instanceof RockFinder) {
$finder = $code;
$finder->debug = true;
// get code of this finder
$code = $finder->getSQL();
// enable debugging now the initial sql request is done
$f->setData($finder);
}
else {
// populate sql property of finder
$f->sql = $code;
}
$form->add($f);
}
$this->config->styles->add('//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css');
$this->config->scripts->add('//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js');
$form->add([
'type' => 'markup',
'value' => "<pre><code>$code</code></pre>",
'label' => 'Resulting SQL',
// 'collapsed' => Inputfield::collapsedYes,
]);
$form->add([
'type' => 'submit',
'id' => 'submit',
'value' => __('Execute SQL'),
'icon' => 'bolt',
]);
return $form->render();
}
}
<?php
$info = [
'title' => 'RockFinder',
'version' => '1.0.7',
'summary' => 'Highly Efficient and Flexible SQL Finder Module to return page data without loading PW pages into memory',
'singular' => true,
'autoload' => false,
'icon' => 'search',
'installs' => ['ProcessRockFinder'],
'requires' => ['ProcessWire>=3.0.97'],
];
\ No newline at end of file
This diff is collapsed.
<?php namespace ProcessWire;
abstract class RockFinderField extends WireData {
public $name;
protected $columns;
public $type;
public $alias;
public $siblingseparator = ':';
// if set to true this will query the current user's language in strict mode
// if set to false, it will return the value of the current language and fallback
// to the default language's value if the current language's value is NULL or empty
public $strictLanguage = false;
public $multiLang = true;
// todo: change parameters to one options array
public function __construct($name, $columns, $type) {
$this->name = $this->alias = $name;
$this->columns = array_merge(['data'], $columns ?: []);
$this->type = $type;
}
/**
* get sql
*/
public function getSql() {
$sql = "SELECT";
$sql .= "\n `pages_id` AS `pageid`";
foreach($this->columns as $column) {
$sql .= ",\n {$this->dataColumn($this->alias)} AS `{$this->fieldAlias($column)}`";
}
$sql .= "\nFROM `field_{$this->name}` AS `$this->alias`";
return $sql;
}
/**
* get array of objects
*/
public function getObjects($limit = 0) {
$limit = $limit ? " LIMIT $limit" : '';
$results = $this->database->query($this->getSQL() . $limit);
return $results->fetchAll(\PDO::FETCH_OBJ);
}
/**
* return the field alias for given column
*/
protected function fieldAlias($column) {
/**
* bug: this does not work for custom fieldnames but makes joined pages with custom alias work
* $finder = new RockFinder("template=person, isFN=1, has_parent=7888", [
* 'lang' => 'bla', // does not work
* 'forename',
* 'surname',
* ]);
* $finder->addField('report', ['pdfs']);
* $finder->addField('report', ['charts'], ['alias'=>'test']);
*/
$alias = $column == 'data'
? "{$this->alias}"
: "{$this->alias}{$this->siblingseparator}$column"
;
return $alias;
}
/**
* select all columns of this fieldtype
*/
public function getJoinSelect() {
$sql = '';
foreach($this->columns as $column) {
$sql .= ",\n `$this->alias`.`{$this->fieldAlias($column)}` AS `{$this->fieldAlias($column)}`";
}
return $sql;
}
/**
* return the select statement for the data column
*/
public function dataColumn($column) {
// if multilang is switched off query "data" column directly
if($this->multiLang == false) return "`$column`.`data`";
// if the field does not support multilang return the data column
$field = $this->fields->get($column);
if(!$field) throw new WireException("Field $column does not exist");
if(!$field->type instanceof FieldtypeLanguageInterface) return "`$column`.`data`";
// multilang is ON, check for the user's language
$lang = $this->wire->user->language;
if($lang != $this->wire->languages->getDefault()) {
// in strict mode we return the language value
if($this->strictLanguage) return "`$column`.`data{$lang->id}`";
// otherwise we return the first non-empty value
else return "COALESCE(NULLIF(`$column`.`data{$lang->id}`, ''), `$column`.`data`)";
}
// user has default language active, return the value
return "`$column`.`data`";
}
/**
* get sql for joining this field's table
*/
public function getJoin() {
return "\n\n/* --- join $this->alias --- */\n".
"LEFT JOIN (" . $this->getSql() . ") AS `$this->alias` ".
"ON `$this->alias`.`pageid` = `pages`.`id`".
"\n/* --- end $this->alias --- */\n";
}
/**
* debugInfo PHP 5.6+ magic method
*
* This is used when you print_r() an object instance.
*
* @return array
*
*/
public function __debugInfo() {
$info = parent::__debugInfo();
$info['name'] = $this->name;
$info['columns'] = $this->columns;
$info['type'] = $this->type;
$info['alias'] = $this->alias;
$info['siblingseparator'] = $this->siblingseparator;
return $info;
}
}
\ No newline at end of file
<?php
$finder = new \ProcessWire\RockFinder('id>0, limit=5', ['title', 'status']);
return $finder->getSQL();
<?php namespace ProcessWire;
class RockFinderFieldClosure extends RockFinderField {
/**
* select all columns of this fieldtype
*/
public function getJoinSelect() {
return ",\n '' AS `{$this->alias}`";
}
/**
* get sql for joining this field's table
*/
public function getJoin() {
// don't join anything
return;
}
}
\ No newline at end of file
![screenshot](../screenshots/imageField.png?raw=true "Screenshot")
\ No newline at end of file
<?php namespace ProcessWire;
class RockFinderFieldFile extends RockFinderField {
public $separator = ',';
/**
* get sql
*/
public function getSql() {
$sql = "SELECT";
$sql .= "\n `pages_id` AS `pageid`";
foreach($this->columns as $column) {
$sql .= ",\n GROUP_CONCAT(`$column` ORDER BY `sort` SEPARATOR '$this->separator') AS `{$this->fieldAlias($column)}`";
}
$sql .= "\nFROM `field_{$this->name}` AS `$this->alias`";
$sql .= "\nGROUP BY `pageid`";
return $sql;
}
}
\ No newline at end of file
![screenshot](../screenshots/pageField2.png?raw=true "Screenshot")
\ No newline at end of file
<?php namespace ProcessWire;
/**
* bug: this does not work for custom fieldnames but makes joined pages with custom alias work
* $finder = new RockFinder("template=person, isFN=1, has_parent=7888", [
* 'lang' => 'bla', // does not work
* 'forename',
* 'surname',
* ]);
* $finder->addField('report', ['pdfs']);
* $finder->addField('report', ['charts'], ['alias'=>'test']);
*/
class RockFinderFieldPage extends RockFinderField {
public $separator = ',';
/**
* get sql
*/
public function getSql() {
if(!$this->columns) return $this->getSqlNoColumns();
else return $this->getSqlWithColumns();
}
/**
* get sql when no additional columns are set
*/
public function getSqlNoColumns() {
$sql = "SELECT";
$sql .= "\n `$this->alias`.`pages_id` AS `pageid`";
$sql .= ",\n GROUP_CONCAT(`$this->alias`.`data` ORDER BY `$this->alias`.`sort` SEPARATOR '$this->separator') AS `$this->alias`";
$sql .= "\nFROM `field_{$this->name}` AS `$this->alias`";
$sql .= "\nGROUP BY `$this->alias`.`pages_id`";
return $sql;
}
/**
* get sql when additional columns are set
* we need to treat that case differently because a group_concat on the data column (the id of the joined page)
* would lead to multiple id entries in the resulting column when a joined field has multiple entries
* for example if we join the page with id 123 having a file field with files 1.jpg and 2.jpg the result would be:
* 123,123 | 1.jpg,2.jpg
*
* bug: joining a page with 2 fields having multiple items but different counts (eg filefield with 2 files
* and files field with 4 files) makes the concat result in wrong returns
*/
public function getSqlWithColumns() {
$sql = "SELECT";
$sql .= "\n `$this->alias`.`pages_id` AS `pageid`";
$sql .= ",\n `$this->alias`.`data` AS `$this->alias`";
foreach($this->columns as $column) {
if($column == 'data') continue;
$sql .= ",\n GROUP_CONCAT({$this->dataColumn($column)} ORDER BY `$this->alias`.`sort` SEPARATOR '$this->separator') AS `$column`";
}
$sql .= "\nFROM `field_{$this->name}` AS `$this->alias`";
// join all fields
foreach($this->columns as $i=>$column) {
if($i==0) continue; // skip data column
else {
// join all following fields based on the pages_id column
$sql .= "\nLEFT JOIN `field_$column` AS `$column` ON `$column`.`pages_id` = `$this->alias`.`data`";
}
}
$sql .= "\nGROUP BY `$this->alias`.`pages_id`";
// if we have additional columns set we also group by the data column
// see description of getSqlWithColumns() method why we need to do this
if($this->columns) $sql .= ", `$this->alias`.`data`";
return $sql;
}
/**
* select all columns of this fieldtype
*/
public function getJoinSelect() {
$sql = '';
foreach($this->columns as $i=>$column) {
if($i==0)
$sql .= ",\n `$this->alias`.`{$this->fieldAlias($column)}` AS `{$this->fieldAlias($column)}`";
else
$sql .= ",\n `$this->alias`.`$column` AS `{$this->fieldAlias($column)}`";
}
return $sql;
}
}
\ No newline at end of file
<?php namespace ProcessWire;
class RockFinderFieldPagestable extends RockFinderField {
/**
* get sql
*/
public function getSql() {
return "SELECT ".
"`id` AS `pageid`, ". // we need the column "pageid" to do the join
"`$this->name` ".
"FROM `pages` AS `$this->alias`";
}
}
\ No newline at end of file
![screenshot](../screenshots/repeaterField.png?raw=true "Screenshot")
\ No newline at end of file
<?php namespace ProcessWire;
class RockFinderFieldRepeater extends RockFinderField {
public $separator = ',';
/**
* get sql
*/
public function getSql() {
$sql = "SELECT";
$sql .= "\n `$this->alias`.`pages_id` AS `pageid`";
$sql .= ",\n `$this->alias`.`data` AS `$this->name`";
foreach($this->columns as $column) {
if($column == 'data') continue;
// todo: data is not multilanguage
$sql .= ",\n GROUP_CONCAT(`$column`.`data` ORDER BY FIND_IN_SET(`$column`.`pages_id`, `$this->alias`.`data`) separator '$this->separator') AS `$column`";
}
$sql .= "\nFROM `field_{$this->name}` AS `$this->alias`";
// join all fields
$ref = '';
foreach($this->columns as $i=>$column) {
if($i==0) continue; // skip data column
if($i==1) {
// the first join needs to be done via find_in_set
// because the data column holds a comma-separated list of page ids
// with all the ids of the repeater pages
$sql .= "\nLEFT JOIN `field_$column` AS `$column` ON FIND_IN_SET(`$column`.`pages_id`, `$this->alias`.`data`)";
$ref = $column;
}
else {
// join all following fields based on the pages_id column
$sql .= "\nLEFT JOIN `field_$column` AS `$column` ON `$column`.`pages_id` = `$ref`.`pages_id`";
}
}
$sql .= "\nGROUP BY `$this->alias`.`pages_id`";
return $sql;
}
/**
* add all fields of this repeater
*/
public function addAllFields() {
$field = $this->wire->fields->get($this->name);
foreach($field->repeaterFields as $id) {
$this->columns[] = $this->wire->fields->get($id)->name;
}
}
/**
* add all provided fields
*/
public function addFields($fields) {
foreach($fields as $field) {
$this->columns[] = $field;
}
}
/**
* select all columns of this fieldtype
*/
public function getJoinSelect() {
$sql = '';
foreach($this->columns as $i=>$column) {
if($i==0)
$sql .= ",\n `$this->alias`.`{$this->fieldAlias($column)}` AS `{$this->fieldAlias($column)}`";
else
$sql .= ",\n `$this->alias`.`$column` AS `{$this->fieldAlias($column)}`";
}
return $sql;
}
}
<?php namespace ProcessWire;
// returns a regular textfield value
class RockFinderFieldText extends RockFinderField {
}
\ No newline at end of file
# RockFinder
## WHY?
This module was built to fill the gap between simple $pages->find() operations and complex SQL queries.
The problem with $pages->find() is that it loads all pages into memory and that can be a problem when querying multiple thousands of pages. Even $pages->findMany() loads all pages into memory and therefore is a lot slower than regular SQL.
The problem with SQL on the other hand is, that the queries are quite complex to build. All fields are separate tables, some repeatable fields use multiple rows for their content that belong to only one single page, you always need to check for the page status (which is not necessary on regular find() operations and therefore nobody is used to that).
In short: It is far too much work to efficiently and easily get an array of data based on PW pages and fields and I need that a lot for my RockGrid module to build all kinds of tabular data.
---
# Basic Usage
## getObjects()
Returns an array of objects.
![screenshot](screenshots/getObjects.png?raw=true "Screenshot")
## getArrays()
Returns an array of arrays.
![screenshot](screenshots/getArrays.png?raw=true "Screenshot")
## getValues()
Returns a flat array of values of the given column/field.
![screenshot](screenshots/getValues.png?raw=true "Screenshot")
## getPages()
Returns PW Page objects.
```php
$finder = new RockFinder('template=invoice, limit=5', ['value', 'date']);
$finder->getPages();
```
By default uses the id column, but another one can be specified:
![screenshot](screenshots/getPages.png?raw=true "Screenshot")
---
# Advanced Usage
## Joins
It is possible to join multiple finders. This is useful whenever you have single
page reference fields and want to show properties of the referenced page. A simple
example could be this join:
```php
$finder1 = new RockFinder('template=rockproject', ['title', 'rockproject_client']);
$finder2 = new RockFinder('template=rockcontact', ['title']);
// join finder
$finder1->join($finder2, 'contact', ['id' => 'rockproject_client']);
```
The syntac is like this:
```php
$baseFinder->join($joinedFinder, 'joinedFinderAlias', ['fieldNameOfJoinedFinder' => 'fieldNameOfBaseFinder']);
```
A more advanced example is this one, joining three finders. Notice that `$finder3`
is manually joined on a column of `$finder2` (having alias `contact`). You can
achieve this by providing not only the field name (then it would join to the base
finder) but also providing the finder-alias and the fieldname manually.
You have to use this syntax for your "fieldname": `{alias}.{alias}_{fieldname}`.
In this example we are joining projects (finder1) and their related clients
(single page reference field of project) and then we join the referral contact
of the project's client based on the "camefrom" id of finder2:
```php
$finder1 = new RockFinder('template=rockproject', [
'title',
'rockproject_client',
]);
$finder2 = new RockFinder('template=rockcontact', [
'title',
'rockcontact_camefrom',
]);
$finder3 = new RockFinder('template=rockcontact', [
'title',
]);
// join finders
$finder1->join($finder2, 'contact', ['id' => 'rockproject_client']);
$finder1->join($finder3, 'referral', ['id' => 'contact.contact_rockcontact_camefrom']);
return $finder1;
```
![complex join](screenshots/join.png)
## Custom SQL: Aggregations, Groupings, Distincts...
You can apply any custom SQL with this technique:
```php
$finder = new RockFinder('template=invoice, limit=0', ['value', 'date']);
$sql = $finder->getSQL();
$finder->sql = "SELECT * FROM ($sql) AS tmp";
d($finder->getObjects());
```
Real example:
```php
$finder = new RockFinder('template=invoice, limit=0', ['value', 'date']);
$sql = $finder->getSQL();
$finder->sql = "SELECT id, SUM(value) AS revenue, DATE_FORMAT(date, '%Y-%m') AS dategroup FROM ($sql) AS tmp GROUP BY dategroup";
d($finder->getObjects());
```
![screenshot](screenshots/groupby.png?raw=true "Screenshot")
Notice that this query takes only 239ms and uses 0.19MB of memory while it queries and aggregates more than 10.000 pages!
You can also add custom SQL queries like this: https://processwire.com/talk/topic/19226-rockfinder-highly-efficient-and-flexible-sql-finder-module/?do=findComment&comment=167804 to easily do "reverse queries", for example show all projects that have the current page selected in a page-reference-field.
## Closures
ATTENTION: This executes a $pages->find() operation on each row, so this makes the whole query significantly slower than without using closures. Closures are a good option if you need to query complex data and only have a very limited number of rows.
![screenshot](screenshots/closures.png?raw=true "Screenshot")
## Querying more complex fields (page reference fields, repeaters, etc)
Querying those fields is not an easy task in SQL because the field's data is spread across several database tables. This data then needs to be joined and you need to make sure that the sort order stays untouched. RockFinder takes care of all that and makes the final query very easy.
See this example of a page reference field called `cats`:
![screenshot](screenshots/pageField.png?raw=true "Screenshot")
The example also shows how you can control the returned content (for example changing the separator symbol). For every supported fieldtype there is a corresponding readme-file in the `fieldTypes` folder of this repo.
You can create custom fieldType-queries by placing a file in `/site/assets/RockFinder`. This makes this module very versatile and you should be able to handle even the most complex edge-case.
---
# Multilanguage
Multilanguage is ON by default. Options:
```php
$finder->strictLanguage = false;
$finder->multiLang = true;
```
moved to github: https://github.com/BernhardBaumrock/RockFinder
\ No newline at end of file