tiki-export_tracker_ajax.php 13.4 KB
Newer Older
1
<?php
2
// (c) Copyright 2002-2011 by authors of the Tiki Wiki CMS Groupware Project
changi67's avatar
changi67 committed
3
// 
4 5
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.
changi67's avatar
changi67 committed
6
// $Id$
7 8 9

@ini_set('max_execution_time', 0); //will not work if safe_mode is on
require_once('tiki-setup.php');
10
require_once('lib/smarty_tiki/modifier.tiki_short_datetime.php');
11
$access->check_feature(array('feature_trackers','feature_ajax'));
12 13 14 15 16 17 18 19 20 21 22 23

if (!isset($_REQUEST['trackerId'])) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->display('error.tpl');
	die;
}

include_once 'tiki-export_tracker_monitor.php';

$monitor_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'_monitor.json';
if (is_file($monitor_filename)) {
	$stat_array = unserialize(file_get_contents($monitor_filename));
24 25 26 27 28
	if ($stat_array['status'] === 'finish') {
		unset($stat_array);
	}
}
if (empty($stat_array)) {
29
	$stat_array = array();
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
	saveStatus(array('user' => $user, 'status' => 'init', 'msg' => 'Starting...'));
}

include_once('lib/trackers/trackerlib.php');

$tracker_info = $trklib->get_tracker($_REQUEST['trackerId']);
if (empty($tracker_info)) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->assign('msg', tra('No tracker indicated'));
	die;
}
if ($t = $trklib->get_tracker_options($_REQUEST['trackerId'])) {
	$tracker_info = array_merge($tracker_info,$t);
}

$tikilib->get_perm_object($_REQUEST['trackerId'], 'tracker', $tracker_info);
46
$access->check_permission('tiki_p_export_tracker');
47 48 49 50

$filters = array();
if (!empty($_REQUEST['listfields'])) {
	if (is_string($_REQUEST['listfields'])) {
51
		$filters['fieldId'] = preg_split('/[,:]/', $_REQUEST['listfields']);
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
	} elseif (is_array($_REQUEST['listfields'])) {
		$filters['fieldId'] = $_REQUEST['listfields'];
	}
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'ls') {
	$filters['or'] = array('isSearchable'=>'y', 'isTblVisible'=>'y');
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'list') {
	$filters['isTblVisible'] = 'y';
}
if ($tiki_p_admin_trackers != 'y') {
	$filters['isHidden'] = array('n', 'c');
}
if ($tiki_p_tracker_view_ratings != 'y') {
	$filters['not'] = array('type'=>'s');
}
$filters['not'] = array('type'=>'h');

$fields = $trklib->list_tracker_fields($_REQUEST['trackerId'], 0, -1, 'position_asc', '', true, $filters);
$listfields = array();
foreach ($fields['data'] as $field) {
	$listfields[$field['fieldId']] = $field;
}

if (!isset($_REQUEST['which'])) {
	$_REQUEST['which'] = 'all';
}
if (!isset($_REQUEST['status'])) {
	$_REQUEST['status'] = '';
}
if (!isset($_REQUEST['initial'])) {
	$_REQUEST['initial'] = '';
}
$filterFields = '';
$values = '';
$exactValues = '';
foreach ($_REQUEST as $key =>$val) {
	if (substr($key, 0, 2) == 'f_' && !empty($val) && (!is_array($val) || !empty($val[0]))) {
		$fieldId = substr($key, 2);
		$filterFields[] = $fieldId;
		if (isset($_REQUEST["x_$fieldId"]) && $_REQUEST["x_$fieldId"] == 't' ) {
			$exactValues[] = '';
92
			$values[] = urldecode($val);
93
		} else {
94
			$exactValues[] = urldecode($val);
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
			$values[] = '';
		}
	}
}

if (isset($_REQUEST['showStatus'])) {
	$showStatus = $_REQUEST['showStatus'] == 'on'? true : false;
} else {
	$showStatus = false;
}

if (isset($_REQUEST['showItemId'])) {
	$showItemId = $_REQUEST['showItemId'] == 'on'? true : false;
} else {
	$showItemId = false;
}

if (isset($_REQUEST['showCreated'])) {
	$showCreated = $_REQUEST['showCreated'] == 'on'? true : false;
} else {
	$showCreated = false;
}

if (isset($_REQUEST['showLastModif'])) {
	$showLastModif = $_REQUEST['showLastModif'] == 'on'? true : false;
} else {
	$showLastModif = false;
}

if (isset($_REQUEST['parse'])) {
	$parse = $_REQUEST['parse'] == 'on'? true : false;
} else {
	$parse = false;
}

if (empty($_REQUEST['encoding'])) {
	$_REQUEST['encoding'] = 'ISO-8859-1';
}
$encoding = $_REQUEST['encoding'];
if (empty($_REQUEST['separator'])) {
	$_REQUEST['separator'] = ',';
}
$separator = $_REQUEST['separator'];
if (empty($_REQUEST['delimitorL'])) {
	$_REQUEST['delimitorL'] = '"';
}
$delimitorL = $_REQUEST['delimitorL'];
if (empty($_REQUEST['delimitorR'])) {
	$_REQUEST['delimitorR'] = '"';
}
$delimitorR = $_REQUEST['delimitorR'];
if (empty($_REQUEST['CR'])) {
	$_REQUEST['CR'] = '%%%';
}
$CR = $_REQUEST['CR'];

$fp = null;
$temp_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'.csv';
if ($_REQUEST['debug']) {
	$fp = fopen($temp_filename, 'w');
155
	echo 'output:'.$temp_filename;
156 157
}

158 159
include_once 'lib/core/Zend/Log.php';
include_once 'lib/core/Zend/Log/Writer/Stream.php';
160 161 162 163 164 165

$writer = new Zend_Log_Writer_Stream($prefs['tmpDir'].'/tracker_export.log');
$logger = new Zend_Log($writer);

$logger->info('------------- start mem used: ' . round(memory_get_usage(true)/1024, 3));

166
saveStatus(array('status' => 'header', 'msg' => '', 'current' => 0));
167

168
session_write_close();
169 170

if (empty($fp)) {
171
	$trklib->write_export_header();
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
}

if ($tracker_info['defaultOrderKey'] == -1)
	$sort_mode = 'lastModif';
elseif ($tracker_info['defaultOrderKey'] == -2)
	$sort_mode = 'created';
elseif ($tracker_info['defaultOrderKey'] == -3)
	$sort_mode = 'itemId';
elseif (isset($tracker_info['defaultOrderKey'])) {
	$sort_mode = 'f_'.$tracker_info['defaultOrderKey'];
} else {
	$sort_mode = 'itemId';
}
if (isset($tracker_info['defaultOrderDir'])) {
	$sort_mode.= "_".$tracker_info['defaultOrderDir'];
} else {
		$sort_mode.= "_asc";
}

function needs_separator($string) {
	return empty($string) || substr($string, strlen($string) - 1, 1) == "\n";
}

// write out header
$str = '';
if ($showItemId) {
	$str .= $delimitorL.'itemId'.$delimitorL;
}
if ($showStatus) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'status'.$delimitorL;
}
if ($showCreated) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'created'.$delimitorL;
}
if ($showLastModif) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'lastModif'.$delimitorL;
}
if (count($listfields) > 0) {
	foreach ($listfields as $field) {
		$str .= needs_separator($str) ? '' : $separator;
		$str .= $delimitorL.$field['name'].' -- '.$field['fieldId'].$delimitorR;
	}
}
218 219 220
if (empty($_REQUEST['encoding']) || $_REQUEST['encoding'] == 'ISO-8859-1') {
	$str = utf8_decode($str);
}
221 222 223 224 225 226 227 228 229
$str .= "\n";
if (!empty($fp)) {
	fwrite($fp, $str);
} else {
	echo $str;
}
$str = '';

$logger->info('header done: ' . round($mem/1024, 4));
230
saveStatus(array('status' => 'header', 'msg' => ''));
231 232 233 234 235 236 237 238

include_once 'lib/filegals/max_upload_size.php';	// sets $memory_limit and $current_memory_usage
$mem = $mem2 = memory_get_usage(true);
// just to get the number of records...
$items = $trklib->list_items($_REQUEST['trackerId'], 0, 0, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues);
$cant = $items['cant'];

if (!empty($_REQUEST['recordsOffset'])) {
239
	$offset = $recordsOffset = ((int) $_REQUEST['recordsOffset']) - 1;
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
} else {
	$offset = $recordsOffset = 0;
}
if (!empty($_REQUEST['recordsMax'])) {
	$recordsMax = $_REQUEST['recordsMax'];
} else {
	$recordsMax = $cant;	// all
}
if ($recordsMax + $offset > $cant) {
	$recordsMax = $cant - $offset;
}

function calc_chunkSize ($keepFree = 1) {
	global $memory_limit, $chunkSize, $recordsMax, $recordsOffset, $offset, $fields;
	if ($memory_limit < 0) {
		$chunkSize = 5000;	// unlimited memory?
	} else {
257 258 259 260 261
		$freeMem = $memory_limit - memory_get_usage(true); 
		if ($freeMem < $keepFree * 1048576) {
			$keepFree = $freeMem / (1048576 * 4);	// leave 25% free in low memory conditions
		}
		$chunkSize = (int)(($freeMem - $keepFree * 1048576 ) / ((count($fields['data']) * 4096 * 6) + (($recordsMax + $recordsOffset - $offset) * 2)));	// combination of possible record size and number of records
262 263
	}
	if ($chunkSize < 10) { $chunkSize = 10; }
264
	if ($chunkSize > 200) { $chunkSize = 200; }
265 266 267 268 269 270 271 272
	if ($chunkSize > $recordsMax) { $chunkSize = $recordsMax; } // limit to request or cant
}

//calc_chunkSize($memory_limit / (3 * 1048576));
calc_chunkSize(10);

$logger->info('$chunkSize: ' . $chunkSize . '    cant: '. $cant . '     fields: ' . count($fields['data']) . '      memlimit: ' . $memory_limit);
$starttime = microtime(true);
273 274
$lasttime = $starttime;

275
$export_cant = 0; $chunks = 0;
276
saveStatus(array('status' => 'waiting', 'total' => $recordsMax + $offset, 'current' => $export_cant, 'msg' => ''));
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297

while (($items = $trklib->list_items($_REQUEST['trackerId'], $offset, $chunkSize, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues)) && !empty($items['data'])) {
	
	// still need to filter the fields that are view only by the admin and the item creator
	if ($tracker_info['useRatings'] == 'y') {
		foreach ($items['data'] as $f=>$v) {
			$items['data'][$f]['my_rate'] = $tikilib->get_user_vote("tracker.".$_REQUEST['trackerId'].'.'.$items['data'][$f]['itemId'],$user);
		}
	}
	
	//$str = '';
	foreach ($items['data'] as $item) {
		if ($showItemId) {
			$str .= $delimitorL.$item['itemId'].$delimitorL;
		}
		if ($showStatus) {
			$str .= needs_separator($str) ? '' : $separator;
			$str .= $delimitorL.$item['status'].$delimitorL;
		}
		if ($showCreated) {
			$str .= needs_separator($str) ? '' : $separator;
298
			$str .= $delimitorL.smarty_modifier_tiki_short_datetime($item['created'], '', 'n').$delimitorL;
299 300 301
		}
		if ($showLastModif) {
			$str .= needs_separator($str) ? '' : $separator;
302
			$str .= $delimitorL.smarty_modifier_tiki_short_datetime($item['lastModif'], '', 'n').$delimitorL;
303
		}
304 305
		if (count($item['field_values']) > 0) {
			foreach ($item['field_values'] as $field_value) {
306
				$data = '';
307
				if ($field_value['isHidden'] == 'n' || $field_value['isHidden'] == 'p' || ($field_value['isHidden'] == 'c' && ($item['itemUser'] == $user || $tiki_p_admin_trackers == 'y')) || ($field_value['isHidden'] == 'y' &&  $tiki_p_admin_trackers == 'y')) {
308 309 310 311 312 313 314 315
					
					// this way seems to be over 5 times slower... not sure why
//					$data = $trklib->get_item_value($item['trackerId'], $item['itemId'],$field_value['fieldId']);
//					if (is_array($data)) {			// TODO handle other types of field better here (preferably in a function in $trklib)
//						$data = implode('%%%', $data);
//					}
//					$data = str_replace(array("\r\n", "\n", '<br />', $delimitorL, $delimitorR), array($CR, $CR, $CR, $delimitorL.$delimitorL, $delimitorR.$delimitorR), $data);
					switch($field_value['type']) {
316
						case 'd': // text etc
317 318
							$data = $field_value['value'];
							if (is_array($data)) {			// TODO handle other types of field better here (preferably in a function in $trklib)
319
								$data = implode($CR, $data);
320
							}
321
							$data = str_replace(array("\r\n", "\n", '<br />', $delimitorL, $delimitorR), array($CR, $CR, $CR, $delimitorL.$delimitorL, $delimitorR.$delimitorR), $data);				
322
							break;
323 324 325
						default: 
							$data = $trklib->get_field_handler($field_value, $item)->renderOutput(array(
								'list_mode' => 'csv', 'CR'=>$CR, 'delimitorL'=>$delimitorL, 'delimitorR'=>$delimitorR
326
						));
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
					}
				}
				$str .= needs_separator($str) ? '' : $separator;
				$str .= $delimitorL.$data.$delimitorR;
			}
		}
		$str .= "\n";
		$export_cant++;
	}

	if (empty($_REQUEST['encoding']) || $_REQUEST['encoding'] == 'ISO-8859-1') {
		$str = utf8_decode($str);
	}
	if (!empty($fp)) {
		fwrite($fp, $str);
	} else {
		echo $str;
	}
345
	saveStatus(array('status' => 'export', 'current' => $export_cant, 'msg' => ''));
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
	$mem2 = memory_get_usage(true);
	
	if ($offset + $chunkSize < $recordsMax + $recordsOffset) {
		$offset += $chunkSize;
	} else {
		break;
	}
	$chunks++;
	$str = '';
	unset($items);
	unset($item);
	calc_chunkSize();		// recalculate chunkSize
	if ($offset + $chunkSize  > $recordsMax + $recordsOffset) {
		$chunkSize = $recordsMax + $recordsOffset - $offset;
	}

362 363
	$leak = $mem2 - $mem;
	$logger->info('done: '.$export_cant.' records - (current chunkSize='.$chunkSize.') - lost mem: '.round(($leak)/1024, 3).' kB - usage total: '.round($mem2/1048576, 3).' MB - time ' . round(microtime(true) - $lasttime,2).' secs');
364
	$mem = memory_get_usage(true);
365 366
	$lasttime = microtime(true);
	
367 368 369 370 371 372 373 374 375 376 377 378 379
	if ($leak + $mem > $memory_limit - 1024 && $chunks > 1) {	// unlikely to work, so fail safely if got past first chunk
		$str = 'Export incomplete, memory limit reached. Exported: '.$export_cant.' out of '.$cant.' records.';
		$logger->info('failed '.$export_cant.' records in ' . round(microtime(true) - $starttime,2).' secs');
		
		saveStatus(array('status' => 'failed', 'current' => $export_cant, 'msg' => 'Export incomplete (probably), memory limit reached.'));
		if (!empty($fp)) {
			fwrite($fp, $str);
		} else {
			echo $str;
		}
		//break;
	}

380
	//time_nanosleep(0, 300);	 // AJAX time!? please?
381 382 383 384 385 386 387
	if (0 && $chunks > 1) {	// debugging
		break;
	}
}

if (!empty($fp)) {
	fclose($fp);
388
	$trklib->write_export_header();
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
	header('Content-Length: ' . filesize($temp_filename));
	readfile($temp_filename);
}
saveStatus(array('status' => 'finish', 'current' => $export_cant));
$logger->info('done '.$export_cant.' records in ' . round(microtime(true) - $starttime,2).' secs');

flush();


// cannot send two outputs it seems :(
//if ($export_cant != $cant) {
//	$msg = tra('Problem with export: Only exported').' '.$export_cant.' '.tra('records.');
//} else {
//	$msg = tra('Exported').' '.$export_cant.' '.tra('records.');
//}
//$redir = $tikiroot.'tiki-view_tracker.php?trackerId='.$_REQUEST['trackerId'].'&cookietab=3&export_result='.urlencode($msg).'export_cant='.$export_cant;
//header('Content-type: text/html; charset: UTF-8');
//header('Content-Disposition: inline');
//header('Location:'.$redir);
die;