Implemented MANTIS-5715 (Add more helpful broken link messages) (0116e47f) · Commits · Composr ecosystem / Composr

lang/EN/critical_error.ini

+1 −0

Original line number	Diff line number	Diff line
		@@ -112,6 +112,7 @@ HTTP_DOWNLOAD_BAD_URL=The specified URL (<kbd>{1}</kbd>) is not valid
		HTTP_DOWNLOAD_STATUS_UNAUTHORIZED=Unauthorized access to URL, <kbd>{1}</kbd>.
		HTTP_DOWNLOAD_STATUS_NOT_FOUND=The URL, <kbd>{1}</kbd>, was not found
		HTTP_DOWNLOAD_STATUS_SERVER_ERROR=Remote server error when reading URL, <kbd>{1}</kbd>.
		HTTP_DOWNLOAD_STATUS_BAD_REQUEST=Bad request when calling URL, <kbd>{1}</kbd>.
		HTTP_DOWNLOAD_STATUS_UNKNOWN=Unknown HTTP status received for <kbd>{1}</kbd> (<kbd>{2}</kbd>).
		MISSING_ADDON=An addon is required (<kbd>{1}</kbd>) which is not installed
		MISSING_MODULE=A module is required (<kbd>{1}</kbd>) which is not installed

lang/EN/health_check.ini

+18 −1

Original line number	Diff line number	Diff line
		@@ -118,7 +118,24 @@ PAGE_INTEGRITY_PROBLEM=Page appears broken, missing closing HTML tag in {1}
		PAGE_SIZE_PROBLEM=Page is very large @ {1} in {2}
		HTTPS_EMBED_PROBLEM=Embedding HTTP resources on HTTPS page: [tt]{1}[/tt] in {2}
		HTTPS_LINKING_PROBLEM=Linking to a local HTTP page on all-HTTPS site: [tt]{1}[/tt] in {2}
		BROKEN_LINK_PROBLEM=Broken link: [tt]{1}[/tt], {2} HTTP response
		BROKEN_LINK_PROBLEM=Broken link: [tt]{1}[/tt], '{2}' code.
		BROKEN_LINK_PROBLEM__401=Broken link: [tt]{1}[/tt], '{2}' code (unauthorized; authorization or logging in to that site is required to view the page)
		BROKEN_LINK_PROBLEM__403=Broken link: [tt]{1}[/tt], '{2}' code (forbidden; access to the page/resource is denied)
		BROKEN_LINK_PROBLEM__404=Broken link: [tt]{1}[/tt], '{2}' code (not found; double-check that the link is correct)
		BROKEN_LINK_PROBLEM__408=Broken link: [tt]{1}[/tt], '{2}' code (request timed out)
		BROKEN_LINK_PROBLEM__410=Broken link: [tt]{1}[/tt], '{2}' code (gone; the page has been removed)
		BROKEN_LINK_PROBLEM__429=Broken link: [tt]{1}[/tt], '{2}' code (too many requests; we have been rate-limited by this site)
		BROKEN_LINK_PROBLEM__500=Broken link: [tt]{1}[/tt], '{2}' code (internal error; there is a problem with this site or its server)
		BROKEN_LINK_PROBLEM__503=Broken link: [tt]{1}[/tt], '{2}' code (service unavailable; the site might be down for maintenance)
		BROKEN_LINK_PROBLEM__curl_error_1=Broken link: [tt]{1}[/tt], '{2}' code (cURL unsupported protocol)
		BROKEN_LINK_PROBLEM__curl_error_6=Broken link: [tt]{1}[/tt], '{2}' code (cURL could not resolve host; double-check that the link is correct)
		BROKEN_LINK_PROBLEM__curl_error_7=Broken link: [tt]{1}[/tt], '{2}' code (cURL could not connect to host or proxy; check proxy settings in the software and that the link is correct)
		BROKEN_LINK_PROBLEM__curl_error_8=Broken link: [tt]{1}[/tt], '{2}' code (cURL could not parse server reply)
		BROKEN_LINK_PROBLEM__curl_error_23=Error processing link: [tt]{1}[/tt], '{2}' code (cURL could not write the contents of the response to file)
		BROKEN_LINK_PROBLEM__curl_error_28=Broken link: [tt]{1}[/tt], '{2}' code (cURL operation timed out)
		BROKEN_LINK_PROBLEM__curl_error_33=Error processing link: [tt]{1}[/tt], '{2}' code (server does not accept byte ranges)
		BROKEN_LINK_PROBLEM__curl_error_47=Broken link: [tt]{1}[/tt], '{2}' code (too many redirects)
		BROKEN_LINK_PROBLEM__curl_error_61=Broken link: [tt]{1}[/tt], '{2}' code (unrecognised content encoding from server)
		INCOMPLETE_CONTENT_PROBLEM=Found a suspicious "{1}" in {2}
		LOCAL_LINKING_PROBLEM=Found links to a local URL in {1}
		FORM_ELEMENT_PROBLEM=Form element(s) found outside of a form in {1}

sources/hooks/systems/health_checks/mistakes_build.php

+14 −5

Original line number	Diff line number	Diff line
		@@ -99,15 +99,17 @@ class Hook_health_check_mistakes_build extends Hook_Health_Check
		$this->stateCheckManual('Do a [url="general check"]https://webhint.io/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');

		$this->stateCheckManual('Test in Mozilla Firefox');
		$this->stateCheckManual('Test in Google Chrome');
		$this->stateCheckManual('Test in IE10');
		$this->stateCheckManual('Test in IE11');
		$this->stateCheckManual('Test in Mozilla Firefox (mobile)');
		$this->stateCheckManual('Test in IE10'); // TODO: remove when we no longer support it
		$this->stateCheckManual('Test in IE11'); // TODO: remove when we no longer support it
		$this->stateCheckManual('Test in Microsoft Edge');
		$this->stateCheckManual('Test in Safari');
		$this->stateCheckManual('Test in Google Chrome');
		$this->stateCheckManual('Test in Google Chrome (mobile)');
		$this->stateCheckManual('Test in Safari');
		$this->stateCheckManual('Test in Safari (mobile)');

		$this->stateCheckManual('Check the website would look good if printed');
		$this->stateCheckManual('Check the website would look good if generated as a PDF');
		}

		/**
		@@ -220,7 +222,14 @@ class Hook_health_check_mistakes_build extends Hook_Health_Check
		*/
		$message = '';
		$ok = check_url_exists($url, 60 * 60 * 24 * 1, true, 3, $message);
		$this->assertTrue($ok, do_lang('BROKEN_LINK_PROBLEM', $url, $message));
		if ($message === null) {
		$message = 'UNKNOWN';
		}
		$message_text = do_lang('BROKEN_LINK_PROBLEM__' . str_replace('-', '_', $message), $url, $message, null, null, false); // Support custom messages for specific status codes
		if ($message_text === null) {
		$message_text = do_lang('BROKEN_LINK_PROBLEM', $url, $message);
		}
		$this->assertTrue($ok, $message_text);
		}
		}

sources/http.php

+52 −11

Original line number	Diff line number	Diff line
		@@ -367,8 +367,16 @@ abstract class HttpDownloader
		public $download_size = null; // ?ID_TEXT. The download size returned from the last HTTP lookup.
		public $download_url = null; // ?ID_TEXT. The URL for the last HTTP lookup.
		public $download_mtime = null; // ?ID_TEXT. The file modification time returned from the last HTTP lookup.
		public $message = null; // string. The status code returned from the last HTTP lookup (e.g. "200" or "404").
		public $message_b = null; // string. The status messagereturned from the last HTTP lookup.

		/**
		* The status code returned.
		* This is usually the HTTP status. But if an error occurred prior to HTTP, it may be something different, like 'curl-error-#'.
		*
		* @var ?string The status code (null: no status code processed yet)
		*/
		public $message = null;

		public $message_b = null; // ?string. The status message returned from the last HTTP lookup.
		public $new_cookies = []; // ?ID_TEXT. The cookies returned from the last HTTP lookup.
		public $filename = null; // ?ID_TEXT. The filename returned from the last HTTP lookup.
		public $charset = null; // ?ID_TEXT. The character set returned from the last HTTP lookup.
		@@ -1205,19 +1213,22 @@ class HttpDownloaderCurl extends HttpDownloader
		return $this->_run($url, $options); // $this->content_size should be set to how many bytes we have so far
		}

		$possible_internal_curl_errors = [1, 2, 4, 5, 16, 34, 35, 41, 43, 45, 48, 52, 53, 54, 55, 56, 58, 59, 60, 64, 66, 77, 80, 81, 82, 83, 89, 90, 91, 92];
		if (!in_array($curl_errno, $possible_internal_curl_errors)) {
		// If any of these error codes are returned, treat as an internal failure on the cURL end, thus we should use a different backend
		$possible_internal_curl_errors = [1, 2, 4, 5, 16, 34, 35, 41, 43, 45, 48, 52, 53, 54, 55, 56, 58, 59, 60, 64, 66, 77, 80, 81, 82, 83, 89, 90, 91, 92, 95, 96, 97, 99, 101];
		if (in_array($curl_errno, $possible_internal_curl_errors)) {
		return false;
		}

		// Probably a request/response error, not a cURL one
		if ($this->trigger_error) {
		warn_exit(protect_from_escaping($error), false, true);
		} else {
		$this->message = 'curl-error-' . protect_from_escaping(strval($curl_errno));
		$this->message_b = protect_from_escaping($error);
		}
		return null;
		}

		return false; // Failed on this backend
		}

		// Response metadata that cURL lets us gather easily
		$this->download_mime_type = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
		if ($this->download_mime_type === false) {
		@@ -1263,6 +1274,7 @@ class HttpDownloaderCurl extends HttpDownloader
		break;

		case '404':
		case '410':
		if (!$this->ignore_http_status) {
		if ($this->trigger_error) {
		warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url)), false, true);
		@@ -1274,7 +1286,19 @@ class HttpDownloaderCurl extends HttpDownloader

		case '400':
		case '429':
		if (!$this->ignore_http_status) {
		if ($this->trigger_error) {
		warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url)), false, true);
		} else {
		$this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url));
		}
		}
		break;

		case '500':
		case '502':
		case '503':
		case '504':
		if (!$this->ignore_http_status) {
		if ($this->trigger_error) {
		warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url)), false, true);
		@@ -1802,6 +1826,7 @@ class HttpDownloaderSockets extends HttpDownloader
		break;

		case '404':
		case '410':
		if (!$this->ignore_http_status) {
		@fclose($mysock);

		@@ -1817,7 +1842,23 @@ class HttpDownloaderSockets extends HttpDownloader

		case '400':
		case '429':
		if (!$this->ignore_http_status) {
		@fclose($mysock);

		if ($this->trigger_error) {
		warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url)), false, true);
		} else {
		$this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url));
		}

		return null;
		}
		break;

		case '500':
		case '502':
		case '503':
		case '504':
		if (!$this->ignore_http_status) {
		@fclose($mysock);

sources/urls2.php

+4 −0

Original line number	Diff line number	Diff line
		@@ -1087,6 +1087,8 @@ function check_url_exists(string $url, ?int $test_freq_secs = null, bool $retry_
		$test1 = $GLOBALS['SITE_DB']->query_select('urls_checked', ['*'], ['url' => $url], 'ORDER BY url_check_time DESC', 1);

		if ((!isset($test1[0])) \|\| ($test1[0]['url_check_time'] < time() - $test_freq_secs) \|\| (($retry_on_failed) && ($test1[0]['url_exists'] == 0))) {
		$old = cms_extend_time_limit($attempts * 6); // Provide a PHP time extension depending on how many attempts we're making

		for ($i = 0; $i < $attempts; $i++) {
		$test2 = cms_http_request($url, ['trigger_error' => false]);
		if (($test2 !== null) && (in_array($test2->message, ['400', '401', '403', '405', '416', '500', '501', '502', '503', '520']))) {
		@@ -1098,6 +1100,8 @@ function check_url_exists(string $url, ?int $test_freq_secs = null, bool $retry_
		}
		}

		cms_set_time_limit($old);

		mark_if_url_exists($url, $exists, ($test2->message === null) ? '' : $test2->message);
		$message = $test2->message;
		$destination_url = $test2->download_url;