Commit 53b2852c authored by Jonny Bradley's avatar Jonny Bradley

[NEW] Tracker dump. Allows an entire tracker to be exported as CSV. Overrides...

[NEW] Tracker dump. Allows an entire tracker to be exported as CSV. Overrides existing permissions so has it's own new perm, and saves raw values only (so far).
Tested with up to 15,000 items with 20 fields each.
Faster with more memory assigned but tested successfully down to around 45M.
parent f36e8f45
......@@ -1290,6 +1290,7 @@ installer/schema/20100222_add_webmail_fromEmail_field_tiki.sql -text
installer/schema/20100222_sheet_toolbar_default_tiki.sql -text
installer/schema/20100223_blog_always_owner_tiki.sql -text
installer/schema/20100224_object_relations_tiki.sql -text
installer/schema/20100226_tracker_dump_perm_tiki.sql -text
installer/schema/index.php -text
installer/shell.php -text
installer/tiki-installer.php -text
......
......@@ -2906,7 +2906,8 @@ INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES('tiki_p_payment_view', 'Can view payment requests and details', 'admin', 'payment', NULL, 'payment_feature');
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES('tiki_p_payment_manual', 'Can enter manual payments', 'admin', 'payment', NULL, 'payment_feature');
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES('tiki_p_payment_request', 'Can request a payment', 'admin', 'payment', NULL, 'payment_feature');
INSERT INTO users_permissions (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES ('tiki_p_admin_modules', 'User can Administer Modules', 'registered', 'tiki', NULL, NULL);
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES ('tiki_p_admin_modules', 'User can Administer Modules', 'registered', 'tiki', NULL, NULL);
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES('tiki_p_tracker_dump', 'Can save a CSV backup of entire trackers', 'admin', 'trackers', NULL, 'feature_trackers');
DROP TABLE IF EXISTS `users_usergroups`;
......
INSERT INTO `users_permissions` (`permName`, `permDesc`, `level`, `type`, `admin`, `feature_check`) VALUES('tiki_p_tracker_dump', 'Can save a CSV backup of entire trackers', 'admin', 'trackers', NULL, 'feature_trackers');
......@@ -1913,6 +1913,123 @@ class TrackerLib extends TikiLib
return $total;
}
function dump_tracker_csv($trackerId) {
global $tikilib;
$tracker_info = $this->get_tracker_options($trackerId);
$fields = $this->list_tracker_fields($trackerId, 0, -1, 'position_asc', '');
$trackerId = (int)$trackerId;
// write out file header
session_write_close();
$this->write_export_header();
// then "field names -- index" as first line
$str = '';
$str .= '"itemId", "status", "created", "lastModif",';
if (count($fields['data']) > 0) {
foreach ($fields['data'] as $field) {
$str .= '"'.$field['name'].' -- '.$field['fieldId'].'",';
}
}
echo $str;
// prepare queries
$mid = ' WHERE tti.`trackerId` = ? ';
$bindvars = array($trackerId);
$join = '';
$query_items = 'SELECT tti.*'
.' FROM `tiki_tracker_items` tti'
.$mid
.' ORDER BY tti.`itemId` ASC';
$query_fields = 'SELECT tti.itemId, ttif.`value`, ttf.`type`'
.' FROM ('
.' `tiki_tracker_items` tti'
.' INNER JOIN `tiki_tracker_item_fields` ttif ON tti.`itemId` = ttif.`itemId`'
.' INNER JOIN `tiki_tracker_fields` ttf ON ttf.`fieldId` = ttif.`fieldId`'
.')'
.$mid
.' ORDER BY tti.`itemId` ASC, ttif.`fieldId` ASC';
$base_tables = '('
.' `tiki_tracker_items` tti'
.' INNER JOIN `tiki_tracker_item_fields` ttif ON tti.`itemId` = ttif.`itemId`'
.' INNER JOIN `tiki_tracker_fields` ttf ON ttf.`fieldId` = ttif.`fieldId`'
.')'.$join;
$query_cant = 'SELECT count(DISTINCT ttif.`itemId`) FROM '.$base_tables.$mid;
$cant = $this->getOne($query_cant, $bindvars);
$avail_mem = $tikilib->get_memory_avail();
$maxrecords_items = intval(($avail_mem - 10 * 1024 * 1025) / 5000); // depends on size of items table (fixed)
$offset_items = 0;
$items = $this->get_dump_items_array($query_items, $bindvars, $maxrecords_items, $offset_items);
$avail_mem = $tikilib->get_memory_avail(); // update avail after getting first batch of items
$maxrecords = (int)($avail_mem / 40000) * count($fields['data']); // depends on number of fields
$canto = $cant * count($fields['data']);
$offset = 0;
$lastItem = -1;
$count = 0; $icount = 0;
$field_values = array();
// write out rows
for ($offset = 0; $offset < $canto; $offset = $offset + $maxrecords) {
$field_values = $this->fetchAll($query_fields, $bindvars, $maxrecords, $offset);
$mem = memory_get_usage(true);
foreach ( $field_values as $res ) {
if ($lastItem != $res['itemId']) {
$lastItem = $res['itemId'];
echo "\n".'"'.$items[$lastItem]['itemId'].'",'.$items[$lastItem]['status'].'",'.$items[$lastItem]['created'].'",'.$items[$lastItem]['lastModif'].'",';
$count++;
$icount++;
if ($icount > $maxrecords_items) {
$offset_items += $maxrecords_items;
$items = $this->get_dump_items_array($query_items, $bindvars, $maxrecords_items, $offset_items);
$icount = 0;
}
}
echo '"' . $res['value'] . '",';
}
ob_flush();
flush();
//if ($offset == 0) { $maxrecords = 1000 * count($fields['data']); }
}
echo "\n";
ob_end_flush();
}
function get_dump_items_array($query, $bindvars, $maxrecords, $offset) {
$items_array = $this->fetchAll($query, $bindvars, $maxrecords, $offset);
$items = array();
foreach ($items_array as $item) {
$items[$item['itemId']] = $item;
}
unset($items_array);
return $items;
}
function write_export_header() {
header("Content-type: text/comma-separated-values; charset:".$_REQUEST['encoding']);
if (!empty($_REQUEST['file'])) {
if (preg_match('/.csv$/', $_REQUEST['file'])) {
$file = $_REQUEST['file'];
} else {
$file = $_REQUEST['file'].'.csv';
}
} else {
$file = tra('tracker').'_'.$_REQUEST['trackerId'].'.csv';
}
header("Content-Disposition: attachment; filename=$file");
header("Expires: 0");
header("Cache-Control: must-revalidate, post-check=0,pre-check=0");
header("Pragma: public");
}
function _describe_category_list($categs) {
global $categlib;
$res = '';
......
......@@ -160,4 +160,36 @@ exportProgress = function () {
{remarksbox type="note" title="Warning"}Please note: Using experimental AJAX export function - work in progress!{/remarksbox}
{/if}
</div>
{if $tiki_p_tracker_dump eq "y"}
<h2>{tr}Dump All Tracker Items{/tr}</h2>
<div>
<form action="{$smarty.server.PHP_SELF}" method="post">
<table class="normal">
<tr class="formcolor">
<td width="20%"><label for="tracker">{tr}Tracker{/tr}</label></td>
<td>
<select name="trackerId" onchange="this.form.submit();" id="dumpTrackerId">
{foreach from=$trackers item=tracker}
<option value="{$tracker.trackerId}" title="{$tracker.description|escape}"{if $tracker.trackerId eq $trackerId} selected="selected"{/if}>
{$tracker.name|escape}
</option>
{/foreach}
</select>
{$recordsMax} {tr}Items{/tr}
</td>
</tr>
</table>
</form>
<form action="tiki-export_tracker.php?trackerId={$trackerId}" method="post" id="dump_form">
<table>
<tr class="formcolor">
<td width="20%">&nbsp;</td>
<td>
<input type="submit" name="dump_tracker" id="dump_tracker" value="{tr}Dump{/tr}" />
</td>
</tr>
</table>
</form>
</div>
{/if}
......@@ -30,6 +30,13 @@ $smarty->assign_by_ref('tracker_info', $tracker_info);
$tikilib->get_perm_object($_REQUEST['trackerId'], 'tracker', $tracker_info);
$access->check_permission('tiki_p_export_tracker');
if (isset($_REQUEST['dump_tracker'])) {
$access->check_permission('tiki_p_tracker_dump');
$trklib->dump_tracker_csv($_REQUEST['trackerId']);
return;
}
$filters = array();
if (!empty($_REQUEST['listfields'])) {
if (is_string($_REQUEST['listfields'])) {
......
......@@ -159,26 +159,10 @@ $logger->info('------------- start mem used: ' . round(memory_get_usage(true)/10
saveStatus(array('status' => 'header', 'msg' => '', 'current' => 0));
function write_export_header() {
header("Content-type: text/comma-separated-values; charset:".$_REQUEST['encoding']);
if (!empty($_REQUEST['file'])) {
if (preg_match('/.csv$/', $_REQUEST['file'])) {
$file = $_REQUEST['file'];
} else {
$file = $_REQUEST['file'].'.csv';
}
} else {
$file = tra('tracker').'_'.$_REQUEST['trackerId'].'.csv';
}
header("Content-Disposition: attachment; filename=$file");
header("Expires: 0");
header("Cache-Control: must-revalidate, post-check=0,pre-check=0");
header("Pragma: public");
}
session_write_close();
if (empty($fp)) {
write_export_header();
$trklib->write_export_header();
}
if ($tracker_info['defaultOrderKey'] == -1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment