Commit fbbbb5d9 authored by Chris Graham's avatar Chris Graham

Various fixes to very broken import timeout refresher

parent ed0afaf9
......@@ -18,6 +18,8 @@
* @package import
*/
declare(ticks=1);
/**
* Module page class.
*/
......@@ -567,17 +569,21 @@ class Module_admin_import
$info = $object->info();
// Protection from if things take too long
$refresh_url = get_self_url(true, false, array('type' => 'import'), true);
$refresh_url = get_self_url(true, false, array('type' => 'import'), false, true);
$refresh_time = either_param_integer('refresh_time', 0); // Shouldn't default, but reported on some systems to do so
if (php_function_allowed('set_time_limit')) {
@set_time_limit($refresh_time);
@set_time_limit($refresh_time); // We cannot rely on this as the timer doesn't count DB activity etc on most PHP platforms but something like a CGI timeout will; hence register_tick_function also below
safe_ini_set('display_errors', '0'); // So that the timeout message does not show, which made the user not think the refresh was going to happen automatically, and could thus result in double-requests
}
send_http_output_ping();
safe_ini_set('log_errors', '0');
global $I_REFRESH_URL;
global $I_REFRESH_URL, $I_REFRESH_TIME;
$I_REFRESH_URL = $refresh_url;
$I_REFRESH_TIME = $refresh_time;
$GLOBALS['NO_QUERY_LIMIT'] = true;
if ($I_REFRESH_TIME != 0) {
register_tick_function('i_timed_refresh');
}
// Get data from session
$session_row = $GLOBALS['SITE_DB']->query_select('import_session', array('*'), array('imp_session' => get_session_id()), '', 1);
......
......@@ -18,6 +18,7 @@ IMPORT_SESSION_EXISTING_REMAP=Resume the previous import session ({1})
IMPORT_SESSION_CURRENT=Continue the current import session ({1})
REFRESH_TIME=Refresh time
DESCRIPTION_REFRESH_TIME=It is not easy for web systems to import large bodies of data, because they work on a page request model. PHP/Browsers may time-out if a page takes too long to load: halting the import. Our import system is re-entrant and automatically reloads: please enter the number of seconds between requests. If you leave it blank we'll try and tell PHP to run forever until completion.
REFRESH_TIMEOUT_REACHED=Refreshing to continue, as the {1} second timeout was reached.
IMPORT_OF_SKIPPED=Import of <kbd>{1}</kbd> is skipped for now because it depends on <kbd>{2}</kbd> to be completed. You should go back and re-import this once the dependency has been satisfied.
IMPORT_NOT_IMPORTED=We cannot continue due to a missing dependency. <kbd>{2}</kbd> of type <kbd>{1}</kbd> is missing. There may be a small corruption in your database &ndash; if you contact us (the developers) with a copy of the database, we may be able to work around it and get you an updated importer.
IMPORT_WARNING=It is recommended that you only import one thing at a time, to reduce the effect of failure. The importer system is designed to be very robust and streamlined, but the wide variety of data presented to it leads to the occasional difficulty. If you find the importer will not import your data, we may be able to assist you; please contact the developers.
......
......@@ -313,7 +313,7 @@ class Hook_vb3
*/
public function import_cns_members($db, $table_prefix, $file_base)
{
$row_start = 0;
$row_start = get_param_integer('JUMPSTART_import_cns_members', 0);
$rows = array();
do {
$rows = $db->query('SELECT *,u.userid AS userid FROM ' . $table_prefix . 'user u LEFT JOIN ' . $table_prefix . 'administrator a ON u.userid=a.userid LEFT JOIN ' . $table_prefix . 'usertextfield t ON u.userid=t.userid ORDER BY u.userid', 200, $row_start);
......@@ -395,6 +395,7 @@ class Hook_vb3
}
$row_start += 200;
$GLOBALS['JUMPSTART_import_cns_members'] = $row_start;
} while (count($rows) > 0);
}
......@@ -409,7 +410,7 @@ class Hook_vb3
{
global $STRICT_FILE;
$row_start = 0;
$row_start = get_param_integer('JUMPSTART_import_cns_member_files', 0);
$rows = array();
do {
$extra = '';
......@@ -459,6 +460,7 @@ class Hook_vb3
}
$row_start += 200;
$GLOBALS['JUMPSTART_import_cns_member_files'] = $row_start;
} while (count($rows) > 0);
}
......@@ -619,7 +621,7 @@ class Hook_vb3
*/
public function import_cns_topics($db, $table_prefix, $file_base)
{
$row_start = 0;
$row_start = get_param_integer('JUMPSTART_import_cns_topics', 0);
$rows = array();
do {
$rows = $db->query('SELECT * FROM ' . $table_prefix . 'thread WHERE visible=1 ORDER BY threadid', 200, $row_start);
......@@ -638,6 +640,7 @@ class Hook_vb3
}
$row_start += 200;
$GLOBALS['JUMPSTART_import_cns_topics'] = $row_start;
} while (count($rows) > 0);
$rows = $db->query('SELECT * FROM ' . $table_prefix . 'announcement ORDER BY announcementid', 200, $row_start);
......@@ -691,7 +694,7 @@ class Hook_vb3
{
global $STRICT_FILE;
$row_start = 0;
$row_start = get_param_integer('JUMPSTART_import_cns_posts', 0);
// Optimisation to speed through quickly, as can be slow scrolling through so many posts we may have already imported!
do {
......@@ -701,6 +704,7 @@ class Hook_vb3
}
$row_start += 200;
$GLOBALS['JUMPSTART_import_cns_posts'] = $row_start;
} while (true);
$rows = array();
......@@ -839,7 +843,7 @@ class Hook_vb3
require_code('attachments2');
require_code('attachments3');
$row_start = 0;
$row_start = get_param_integer('JUMPSTART_import_cns_post_files', 0);
$rows = array();
do {
$rows = $db->query('SELECT * FROM ' . $table_prefix . 'attachment ORDER BY attachmentid', 200, $row_start);
......@@ -875,6 +879,7 @@ class Hook_vb3
}
$row_start += 200;
$GLOBALS['JUMPSTART_import_cns_post_files'] = $row_start;
} while (count($rows) > 0);
}
......
......@@ -172,7 +172,22 @@ function find_derivative_filename($dir, $file, $shun_gif = false)
}
/**
* Force a page refresh due to maximum execution timeout.
* Detect if we have reached our timerout, more accurately than set_time_limit could.
*/
function i_timed_refresh()
{
static $time = null;
if ($time === null) {
$time = time();
}
global $I_REFRESH_TIME;
if (time() >= $time + $I_REFRESH_TIME) {
i_force_refresh();
}
}
/**
* Force a page refresh due to maximum execution timeout / tick-function determined timeout.
*/
function i_force_refresh()
{
......@@ -181,8 +196,23 @@ function i_force_refresh()
log_hack_attack_and_exit('HEADER_SPLIT_HACK');
}
require_code('site2');
smart_redirect($GLOBALS['I_REFRESH_URL']);
global $I_REFRESH_URL, $I_REFRESH_TIME;
$post = build_keep_post_fields(null, true);
$refresh = do_template('JS_REFRESH', array('FORM_NAME' => 'redir_form'));
$title = get_screen_title('IMPORT');
$url = $I_REFRESH_URL;
foreach ($GLOBALS as $key => $val) {
if (preg_match('#^JUMPSTART_#', $key) != 0) {
$url .= '&' . $key . '=' . strval($val);
}
}
$middle = do_template('REDIRECT_POST_METHOD_SCREEN', array('REFRESH' => $refresh, 'TITLE' => $title, 'TEXT' => do_lang_tempcode('REFRESH_TIMEOUT_REACHED', strval($I_REFRESH_TIME)), 'URL' => $url, 'POST' => $post));
$echo = globalise($middle, null, '', true);
$echo->evaluate_echo();
exit();
}
}
......
......@@ -3713,6 +3713,7 @@ function string_scan($lang, $do_guessing = false, $only_exists = true)
DESCRIPTION_REDO_TIME
DESCRIPTION_REFERRER_SHARE
DESCRIPTION_REFRESH_TIME
REFRESH_TIMEOUT_REACHED
DESCRIPTION_REGIONALITY_SHARE
DESCRIPTION_RELEVANCE_LEVEL
DESCRIPTION_REMEMBER_PASSWORD
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment